diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 474b8327c..fb006874b 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -277,6 +277,21 @@ Cursor-specific model parameters: Explicit variants send Cursor's `default` model with its `optimization` parameter, preserving the selection on every request. They remain available when live discovery omits `default`. +### Vision + +Native Cursor vision uses `SelectedImage` (JPEG soft-cap + `blobIdWithData`) for models that can see +images natively — Claude, Gemini, GPT, Kimi, and Grok among them. Auto, `composer-*`, and GLM +(`glm-5.2`) stay on the curated `noVisionModels` list and use the vision describe sidecar instead. +Trailing `` developer injections (Codex Desktop collab guidance after `view_image`) +are transparent for SelectedImage promotion so the continuation still carries the image and +promote nudge. + +After pulling Cursor vision fixes, run `ocx ensure` so the proxy PID is the workspace `src/cli` +binary rather than a stale install. Stale `providers.cursor.noVisionModels` stamps that list every +Cursor model are healed back to the curated Auto/Composer/GLM set on OAuth reconcile. For +`cursor/grok-4.5`, Codex effort `none`/`minimal` maps to wire tier `medium` (some plans reject +`-low` with Connect `not_found`); explicit `low` still passes through when the account exposes it. + Cursor server-driven local tools are disabled by default. Codex continues using its own tools such as `apply_patch` and `exec_command` with its own approval and sandbox policy: diff --git a/src/adapters/cursor.ts b/src/adapters/cursor.ts index 015eac1cf..35e1065b1 100644 --- a/src/adapters/cursor.ts +++ b/src/adapters/cursor.ts @@ -7,6 +7,7 @@ import { isCursorBenignCancelError, isCursorInvalidArgumentError, safeCursorErro import { isCursorExternalWireModel } from "./cursor/discovery"; import { createCursorKvStore, type CursorKvStore } from "./cursor/kv-store"; import { mapCursorServerMessage } from "./cursor/message-mapper"; +import { cursorIsTrailingToolResultContinuation } from "./cursor/images"; import { createCursorRequest } from "./cursor/request-builder"; import { createLiveCursorTransport, @@ -111,7 +112,9 @@ export function createCursorAdapter(provider: OcxProviderConfig, deps: CursorAda _parsed._cursorConversationId = request.conversationId; let emittedOutput = false; let replayUnsafe = false; - const lastRawIsToolResult = _parsed.context.messages.at(-1)?.role === "toolResult"; + // Desktop multi_agent developer suffixes trail toolResult; treat those as continuations + // so invalid_argument does not force a fresh conversation mid tool resume. + const lastRawIsToolResult = cursorIsTrailingToolResultContinuation(_parsed.context.messages); const runOnce = async (activeRequest: ReturnType) => { await runCursorTurnWithRetry( diff --git a/src/adapters/cursor/discovery.ts b/src/adapters/cursor/discovery.ts index 747fb32d1..e7dd96691 100644 --- a/src/adapters/cursor/discovery.ts +++ b/src/adapters/cursor/discovery.ts @@ -103,6 +103,23 @@ export const CURSOR_ROUTER_MODEL_IDS = [ ...CURSOR_ROUTING_LEVELS.map(level => `${CURSOR_AUTO_MODEL_ID}-${level}`), ] as const; +/** + * Cursor models that cannot see images natively. OpenCodex routes them through the vision + * sidecar (option B: catalog still advertises image so Codex can attach). Evidence: + * - Composer family (`composer-*`): Cursor staff — text-only; "Model does not support images" + * - Auto / router modes: Cursor docs omit Images for Auto Cost; staff — pick Claude/GPT for images + * - glm-5.2: Cursor docs omit Images; Z.ai GLM-5.2 is text-only (vision is GLM-5V) + * + * `composer-*` uses modelInList's trailing-`*` prefix match so a new Composer slug stays sidecar + * until proven multimodal. Everyone else in the static seed (Claude, Gemini, GPT, Kimi, Grok) + * takes SelectedImage. Other live-discovered ids stay unclassified (native path) until curated. + */ +export const CURSOR_NO_VISION_MODELS = [ + ...CURSOR_ROUTER_MODEL_IDS, + "composer-*", + "glm-5.2", +] as const; + /** Wire id Cursor Connect expects for the auto-router (GetUsableModels returns `default`, not `auto`). */ export const CURSOR_AUTO_WIRE_MODEL_ID = "default"; diff --git a/src/adapters/cursor/effort-map.ts b/src/adapters/cursor/effort-map.ts index 1e937b310..93c7a62d2 100644 --- a/src/adapters/cursor/effort-map.ts +++ b/src/adapters/cursor/effort-map.ts @@ -86,11 +86,19 @@ function codexEffortRank(reasoning: string | undefined): "low" | "medium" | "hig /** * The Cursor effort suffix to use for `baseModelId` given a Codex reasoning effort, or `undefined` when * the model takes no suffix (bare). Literal model tiers pass through; unknown efforts clamp by rank. + * + * Grok 4.5 special case: Codex `none`/`minimal` map to `medium`, not `low`. Cursor Start fixes Grok + * at medium and live Connect returns `not_found` for `grok-4.5-low` on some plans; explicit `low` + * still passes through for Pro accounts that expose it. */ export function cursorEffortSuffix(baseModelId: string, reasoning: string | undefined): string | undefined { const tiers = CURSOR_MODEL_EFFORT_TIERS[baseModelId]; if (!tiers || tiers.length === 0) return undefined; const requested = normalizeRequestedEffort(reasoning); + const isGrok45 = baseModelId === "grok-4.5" || baseModelId === "grok-4.5-fast"; + if (isGrok45 && (requested === "none" || requested === "minimal") && tiers.includes("medium")) { + return "medium"; + } if (requested && tiers.includes(requested)) return requested; switch (codexEffortRank(reasoning)) { case "low": diff --git a/src/adapters/cursor/images.ts b/src/adapters/cursor/images.ts new file mode 100644 index 000000000..15708ae90 --- /dev/null +++ b/src/adapters/cursor/images.ts @@ -0,0 +1,929 @@ +import { randomUUID } from "node:crypto"; +import { create } from "@bufbuild/protobuf"; +import type { OcxContentPart, OcxImageContent, OcxMessage } from "../../types"; +import { assessUrlDestination, resolvePublicAddresses } from "../../lib/destination-policy"; +import { pinnedHttpsGet } from "../../images/artifacts"; +import type { PinnedAddress } from "../../lib/pinned-http"; +import { + SelectedContextSchema, + SelectedImageSchema, + SelectedImage_BlobIdWithDataSchema, + SelectedImage_DimensionSchema, + type SelectedContext, + type SelectedImage, +} from "./gen/agent_pb"; +import { + storeCursorBlob, + type CursorBlobRequestScopeToken, +} from "./native-exec"; + +/** Final per-image byte cap after prep (OmniRoute / composer-api style). */ +export const MAX_CURSOR_IMAGE_BYTES = 1024 * 1024; + +/** + * Inbound decode/fetch bomb ceiling before JPEG prep. Large clipboard PNGs may exceed + * {@link MAX_CURSOR_IMAGE_BYTES} raw but shrink under the wire cap after re-encode. + */ +export const MAX_CURSOR_IMAGE_DECODE_BYTES = 16 * 1024 * 1024; + +/** + * Soft target for Cursor vision hydration. Live A/B: ~430 KiB PNG failed ("gray"/wrong UI) + * while the same visual as ~75 KiB JPEG succeeded. Prefer JPEG at or under this size. + */ +export const CURSOR_VISION_SOFT_MAX_BYTES = 100 * 1024; + +/** Soft target when the client requests `detail: original` or `high`. */ +export const CURSOR_VISION_SOFT_MAX_BYTES_HIGH = 256 * 1024; + +/** Longest edge after Cursor vision prep (Cursor staff guidance: ≤ 2000 px). */ +export const CURSOR_VISION_MAX_EDGE = 2000; + +/** + * Decode bomb: reject images whose sniffed longest edge exceeds this before Bun.Image. + * Separate from {@link CURSOR_VISION_MAX_EDGE} (output resize target). + */ +export const MAX_CURSOR_IMAGE_DECODE_EDGE = 8192; + +/** Decode bomb: reject images whose sniffed pixel count exceeds this before Bun.Image. */ +export const MAX_CURSOR_IMAGE_PIXELS = 25_000_000; + +const CURSOR_VISION_JPEG_QUALITIES_DEFAULT = [85, 70, 55, 40] as const; +const CURSOR_VISION_JPEG_QUALITIES_HIGH = [90, 80, 65, 50] as const; +/** Stop shrinking below this longest edge when chasing the soft byte cap. */ +const CURSOR_VISION_SOFT_MIN_EDGE = 256; +const CURSOR_VISION_SOFT_SHRINK = 0.85; + +const CURSOR_VISION_PASSTHROUGH_MIME = new Set([ + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/webp", +]); + +/** Upper bound on images attached to one Cursor turn. */ +export const MAX_CURSOR_IMAGES = 12; + +/** + * UserMessage text when promoting view_image tool-result pixels onto SelectedImage. + * Empty text alone is accepted for attach-only turns; tool promotions use a nudge so + * Cursor/Grok do not invent content from file paths. + */ +export const CURSOR_VISION_PROMOTE_NUDGE = + "Describe the image from the tool result. Do not infer content from file paths or names."; + +/** Honest marker when MCP tool-result image bytes are peeled onto SelectedImage instead. */ +export const CURSOR_VISION_MCP_IMAGE_OMITTED = "[image attached via SelectedImage]"; + +/** Marker when an image cannot be prepared for the Cursor vision wire. */ +export const CURSOR_VISION_IMAGE_OMITTED = + "[image omitted: undecodable or unsupported type]"; + +const IMAGE_FETCH_TIMEOUT_MS = (() => { + const parsed = Number.parseInt(process.env.CURSOR_IMAGE_FETCH_TIMEOUT_MS || "15000", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 15_000; +})(); + +/** Aggregate deadline for prepare+resolve before the Cursor stream opens. */ +export const CURSOR_IMAGE_PHASE_TIMEOUT_MS = (() => { + const parsed = Number.parseInt(process.env.CURSOR_IMAGE_PHASE_TIMEOUT_MS || "30000", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 30_000; +})(); + +/** Part key for per-image SelectedImage promotion omit (`callId` + image part index). */ +export function cursorVisionImagePartKey(callId: string, imagePartIndex: number): string { + return `${callId}#${imagePartIndex}`; +} + +export function createCursorImagePhaseSignal(parent?: AbortSignal): { + signal: AbortSignal; + cancel: () => void; +} { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), CURSOR_IMAGE_PHASE_TIMEOUT_MS); + const cancel = () => { + clearTimeout(timer); + }; + if (parent) { + if (parent.aborted) controller.abort(); + else { + parent.addEventListener("abort", () => { + cancel(); + controller.abort(); + }, { once: true }); + } + } + controller.signal.addEventListener("abort", cancel, { once: true }); + return { signal: controller.signal, cancel }; +} +export class CursorImageError extends Error { + readonly status: number; + + constructor(message: string, status = 400) { + super(message); + this.name = "CursorImageError"; + this.status = status; + } +} + +export interface ResolvedCursorImage { + data: Uint8Array; + mimeType: string; + uuid: string; + /** Codex/OpenAI image detail hint; affects JPEG soft-cap tier. */ + detail?: string; +} + +export type PrepareCursorImageOutcome = + | { status: "ready"; image: ResolvedCursorImage } + | { status: "omitted"; reason: string }; + +function isImagePart(part: OcxContentPart): part is OcxImageContent { + return part.type === "image"; +} + +function estimatedBase64DecodedBytes(payload: string): number { + return Math.floor((payload.length * 3) / 4); +} + +function isHighDetail(detail: string | undefined): boolean { + const normalized = (detail ?? "").trim().toLowerCase(); + return normalized === "original" || normalized === "high"; +} + +function softMaxBytesForDetail(detail: string | undefined): number { + return isHighDetail(detail) ? CURSOR_VISION_SOFT_MAX_BYTES_HIGH : CURSOR_VISION_SOFT_MAX_BYTES; +} + +function jpegQualitiesForDetail(detail: string | undefined): readonly number[] { + return isHighDetail(detail) ? CURSOR_VISION_JPEG_QUALITIES_HIGH : CURSOR_VISION_JPEG_QUALITIES_DEFAULT; +} + +export function decodeCursorImageDataUrl(url: string): { data: Uint8Array; mimeType: string } { + const comma = url.indexOf(","); + if (comma < 0) throw new CursorImageError("Image data URL is malformed."); + const header = url.slice(5, comma); + const payload = url.slice(comma + 1); + const isBase64 = /;base64/i.test(header); + const mimeType = (header.split(";")[0] || "").trim().toLowerCase() || "application/octet-stream"; + + if (!mimeType.startsWith("image/")) { + throw new CursorImageError("Image data URL must have an image/* media type."); + } + if (!isBase64) { + throw new CursorImageError("Image data URL must be base64-encoded."); + } + if (payload.length > MAX_CURSOR_IMAGE_DECODE_BYTES * 2) { + throw new CursorImageError("Image input is too large to process safely."); + } + + const normalized = payload.replace(/\s/g, ""); + if (normalized.length === 0) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + // Reject lenient Buffer.from acceptances (wrong alphabet, bad padding, truncated groups). + if (normalized.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(normalized)) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (estimatedBase64DecodedBytes(normalized) > MAX_CURSOR_IMAGE_DECODE_BYTES) { + throw new CursorImageError("Image input is too large to process safely."); + } + + let data: Uint8Array; + try { + data = Buffer.from(normalized, "base64"); + } catch { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (data.byteLength === 0) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + // Round-trip guard: Node/Bun can silently drop trailing garbage. + if (Buffer.from(data).toString("base64").replace(/=+$/, "") !== normalized.replace(/=+$/, "")) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (data.byteLength > MAX_CURSOR_IMAGE_DECODE_BYTES) { + throw new CursorImageError("Image input is too large to process safely."); + } + return { data, mimeType }; +} + +function pickPinnedAddress(addresses: PinnedAddress[]): PinnedAddress { + return addresses.find(address => address.family === 4) ?? addresses[0]!; +} + +async function fetchHttpsImageBytes(url: string, signal?: AbortSignal): Promise<{ data: Uint8Array; mimeType: string }> { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new CursorImageError("Image URL is invalid."); + } + if (parsed.protocol !== "https:") { + throw new CursorImageError("Image URL must use HTTPS."); + } + + const assessment = assessUrlDestination(url); + if (assessment && assessment.kind !== "public" && assessment.kind !== "hostname") { + throw new CursorImageError("Image URL points to a blocked address."); + } + + let resolved: Awaited>; + try { + resolved = await resolvePublicAddresses(url, { context: "Cursor image" }); + } catch { + throw new CursorImageError("Image URL host could not be resolved."); + } + + const controller = new AbortController(); + const onAbort = () => controller.abort(); + if (signal) { + if (signal.aborted) controller.abort(); + else signal.addEventListener("abort", onAbort, { once: true }); + } + const timer = setTimeout(() => controller.abort(), IMAGE_FETCH_TIMEOUT_MS); + + try { + const response = await pinnedHttpsGet(url, pickPinnedAddress(resolved.addresses), controller.signal, { + maxBytes: MAX_CURSOR_IMAGE_DECODE_BYTES, + }); + const contentType = (response.headers.get("content-type") || "").toLowerCase(); + const mimeType = contentType.split(";")[0]?.trim() || ""; + if (!mimeType.startsWith("image/")) { + throw new CursorImageError("Image URL did not return an image content type."); + } + if (!response.body) throw new CursorImageError("Image URL returned no body."); + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_CURSOR_IMAGE_DECODE_BYTES) { + throw new CursorImageError("Image input is too large to process safely."); + } + chunks.push(value); + } + } finally { + try { await reader.cancel(); } catch { /* ignore */ } + reader.releaseLock(); + } + + const data = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + data.set(chunk, offset); + offset += chunk.byteLength; + } + return { data, mimeType }; + } catch (error) { + if (error instanceof CursorImageError) throw error; + // Preserve AbortError so the image-phase deadline is not soft-omitted as a fetch failure. + if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) throw error; + throw new CursorImageError("Could not fetch the image URL."); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + } +} + +function throwIfImagePhaseAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + if (signal.reason instanceof Error) throw signal.reason; + const err = new Error("Cursor image phase aborted"); + err.name = "AbortError"; + throw err; +} + +/** Magic-byte format sniff (independent of declared MIME). */ +export function sniffCursorImageFormat( + data: Uint8Array, +): "png" | "jpeg" | "gif" | "webp" | undefined { + if ( + data.byteLength >= 8 + && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47 + && data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a + ) { + return "png"; + } + if ( + data.byteLength >= 6 + && data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x38 + ) { + return "gif"; + } + if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) return "jpeg"; + if ( + data.byteLength >= 12 + && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 + && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50 + ) { + return "webp"; + } + return undefined; +} + +/** Collect image URLs from one message's content parts, preserving order. */ +export function extractCursorImageUrls(content: string | readonly OcxContentPart[]): string[] { + return extractCursorImageParts(content).map(part => part.imageUrl); +} + +export interface CursorImagePartRef { + imageUrl: string; + detail?: string; +} + +/** Collect image parts (URL + optional detail) from one message's content. */ +export function extractCursorImageParts( + content: string | readonly OcxContentPart[], +): CursorImagePartRef[] { + if (typeof content === "string" || !Array.isArray(content)) return []; + const parts: CursorImagePartRef[] = []; + for (const part of content) { + if (isImagePart(part) && typeof part.imageUrl === "string" && part.imageUrl.length > 0) { + parts.push({ + imageUrl: part.imageUrl, + ...(typeof part.detail === "string" && part.detail.length > 0 ? { detail: part.detail } : {}), + }); + } + } + return parts; +} + +/** + * Resolve OpenCodex image parts (data: or https:) into bytes for SelectedImage. + * Prep (JPEG soft-cap) runs before the 1 MiB wire cap so large clipboard PNGs can shrink. + * Unsupported / undecodable images are omitted (fail-closed). + */ +export async function resolveCursorImages( + imageUrls: readonly string[], + signal?: AbortSignal, + options?: { details?: readonly (string | undefined)[] }, +): Promise { + if (imageUrls.length > MAX_CURSOR_IMAGES) { + throw new CursorImageError(`Too many images in one request (max ${MAX_CURSOR_IMAGES}).`); + } + + const out: ResolvedCursorImage[] = []; + for (let i = 0; i < imageUrls.length; i++) { + throwIfImagePhaseAborted(signal); + const url = imageUrls[i]; + if (typeof url !== "string" || url.length === 0) { + // Soft-omit missing URLs rather than aborting a mixed turn. + continue; + } + try { + const resolved = url.toLowerCase().startsWith("data:") + ? decodeCursorImageDataUrl(url) + : await fetchHttpsImageBytes(url, signal); + if (resolved.data.byteLength === 0) continue; + const outcome = await prepareCursorImageForWire({ + data: resolved.data, + mimeType: resolved.mimeType, + uuid: randomUUID(), + ...(options?.details?.[i] ? { detail: options.details[i] } : {}), + }, signal); + if (outcome.status === "omitted") continue; + if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) continue; + out.push(outcome.image); + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + if (err instanceof CursorImageError) continue; + continue; + } + } + return out; +} + +export async function resolveCursorImageParts( + parts: readonly CursorImagePartRef[], + signal?: AbortSignal, +): Promise { + return resolveCursorImages( + parts.map(part => part.imageUrl), + signal, + { details: parts.map(part => part.detail) }, + ); +} + +/** Filename Cursor clients typically put on SelectedImage.path (shunt / agent parity). */ +export function cursorImageAttachmentPath(uuid: string, mimeType: string): string { + const normalized = mimeType.toLowerCase(); + const ext = normalized === "image/jpeg" || normalized === "image/jpg" ? "jpg" + : normalized === "image/gif" ? "gif" + : normalized === "image/webp" ? "webp" + : "png"; + return `attachment-${uuid}.${ext}`; +} + +/** + * Re-encode toward a JPEG under the soft vision cap when Bun can decode the payload. + * Unsupported MIME, oversize dimensions/pixels, or undecodable bytes are omitted (fail-closed). + * After the quality ladder, edges shrink iteratively until the soft byte cap is met + * (or the min edge floor is hit) so large clipboard PNGs do not leave >softMax JPEGs + * that Cursor vision hallucinates on. + */ +export async function prepareCursorImageForWire( + image: ResolvedCursorImage, + signal?: AbortSignal, +): Promise { + throwIfImagePhaseAborted(signal); + const mime = image.mimeType.toLowerCase(); + const softMax = softMaxBytesForDetail(image.detail); + const qualities = jpegQualitiesForDetail(image.detail); + const lowestQuality = qualities[qualities.length - 1]!; + + if (!CURSOR_VISION_PASSTHROUGH_MIME.has(mime)) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + + const format = sniffCursorImageFormat(image.data); + // Peek headers before Bun.Image so huge compressed bombs fail closed cheaply. + const sniffed = sniffCursorImageDimensions(image.data); + if (sniffed) { + const edge = Math.max(sniffed.width, sniffed.height); + const pixels = sniffed.width * sniffed.height; + if (edge > MAX_CURSOR_IMAGE_DECODE_EDGE || pixels > MAX_CURSOR_IMAGE_PIXELS) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + } + + // Second-pass skip: already soft-capped JPEG that has a real SOF (not SOI-only junk). + const declaredJpeg = mime === "image/jpeg" || mime === "image/jpg"; + const alreadySmallJpeg = declaredJpeg + && format === "jpeg" + && sniffed !== undefined + && image.data.byteLength <= softMax; + if (alreadySmallJpeg) { + return { status: "ready", image }; + } + + try { + throwIfImagePhaseAborted(signal); + // Force a full decode before accepting passthrough / encode (Anthropic-style validate). + await new Bun.Image(image.data).resize(1, 1).jpeg({ quality: 1 }).toBuffer(); + + // Passthrough only when declared MIME matches actual JPEG magic (never PNG-as-JPEG). + if (declaredJpeg && format === "jpeg" && image.data.byteLength <= softMax) { + return { status: "ready", image }; + } + + throwIfImagePhaseAborted(signal); + const meta = await new Bun.Image(image.data).metadata(); + const width = typeof meta.width === "number" ? meta.width : 0; + const height = typeof meta.height === "number" ? meta.height : 0; + if (width > 0 && height > 0) { + const edge = Math.max(width, height); + if (edge > MAX_CURSOR_IMAGE_DECODE_EDGE || width * height > MAX_CURSOR_IMAGE_PIXELS) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + } + let targetW = width; + let targetH = height; + if (width > 0 && height > 0 && Math.max(width, height) > CURSOR_VISION_MAX_EDGE) { + const scale = CURSOR_VISION_MAX_EDGE / Math.max(width, height); + targetW = Math.max(1, Math.round(width * scale)); + targetH = Math.max(1, Math.round(height * scale)); + } + + const encodeAt = async (w: number, h: number, quality: number): Promise => { + throwIfImagePhaseAborted(signal); + let pipeline = new Bun.Image(image.data); + if (w > 0 && h > 0 && (w !== width || h !== height)) { + pipeline = pipeline.resize(w, h); + } + return new Uint8Array(await pipeline.jpeg({ quality }).bytes()); + }; + + let best: Uint8Array | undefined; + for (const quality of qualities) { + const encoded = await encodeAt(targetW, targetH, quality); + if (!best || encoded.byteLength < best.byteLength) best = encoded; + if (encoded.byteLength <= softMax) { + return { + status: "ready", + image: { ...image, data: encoded, mimeType: "image/jpeg" }, + }; + } + } + + // Quality ladder missed the soft cap — shrink edges until it fits or we hit the floor. + while ( + best + && best.byteLength > softMax + && targetW > 0 + && targetH > 0 + && Math.max(targetW, targetH) > CURSOR_VISION_SOFT_MIN_EDGE + ) { + throwIfImagePhaseAborted(signal); + const nextW = Math.max(1, Math.round(targetW * CURSOR_VISION_SOFT_SHRINK)); + const nextH = Math.max(1, Math.round(targetH * CURSOR_VISION_SOFT_SHRINK)); + if (Math.max(nextW, nextH) < CURSOR_VISION_SOFT_MIN_EDGE) { + const scale = CURSOR_VISION_SOFT_MIN_EDGE / Math.max(targetW, targetH); + targetW = Math.max(1, Math.round(targetW * scale)); + targetH = Math.max(1, Math.round(targetH * scale)); + } else { + targetW = nextW; + targetH = nextH; + } + const encoded = await encodeAt(targetW, targetH, lowestQuality); + if (!best || encoded.byteLength < best.byteLength) best = encoded; + if (encoded.byteLength <= softMax) { + return { + status: "ready", + image: { ...image, data: encoded, mimeType: "image/jpeg" }, + }; + } + if (Math.max(targetW, targetH) <= CURSOR_VISION_SOFT_MIN_EDGE) break; + } + + if (best) { + return { + status: "ready", + image: { ...image, data: best, mimeType: "image/jpeg" }, + }; + } + // Undeclared/mismatched magic with no encode result — omit rather than lie about MIME. + if (declaredJpeg && format !== "jpeg") { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + return { status: "ready", image }; + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } +} + +/** + * Sniff PNG/JPEG/GIF/WebP dimensions from raw bytes when the header is present. + * Best-effort only — unknown formats return undefined (dimension is optional). + */ +export function sniffCursorImageDimensions( + data: Uint8Array, +): { width: number; height: number } | undefined { + // PNG: signature + IHDR chunk (width/height at bytes 16..23) + if ( + data.byteLength >= 24 + && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47 + && data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a + ) { + const width = ((data[16]! << 24) | (data[17]! << 16) | (data[18]! << 8) | data[19]!) >>> 0; + const height = ((data[20]! << 24) | (data[21]! << 16) | (data[22]! << 8) | data[23]!) >>> 0; + if (width > 0 && height > 0) return { width, height }; + } + // GIF: "GIF8" + width/height as little-endian u16 at bytes 6..9 + if ( + data.byteLength >= 10 + && data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x38 + ) { + const width = data[6]! | (data[7]! << 8); + const height = data[8]! | (data[9]! << 8); + if (width > 0 && height > 0) return { width, height }; + } + // WebP: RIFF....WEBP + VP8X / VP8 / VP8L (same layout as anthropic-image-guard). + if ( + data.byteLength >= 30 + && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 + && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50 + ) { + const fourcc = String.fromCharCode(data[12]!, data[13]!, data[14]!, data[15]!); + if (fourcc === "VP8X") { + const width = 1 + (data[24]! | (data[25]! << 8) | (data[26]! << 16)); + const height = 1 + (data[27]! | (data[28]! << 8) | (data[29]! << 16)); + if (width > 0 && height > 0) return { width, height }; + } else if (fourcc === "VP8 ") { + if (data[23] === 0x9d && data[24] === 0x01 && data[25] === 0x2a) { + const width = (data[26]! | (data[27]! << 8)) & 0x3fff; + const height = (data[28]! | (data[29]! << 8)) & 0x3fff; + if (width > 0 && height > 0) return { width, height }; + } + } else if (fourcc === "VP8L" && data[20] === 0x2f) { + const raw = data[21]! | (data[22]! << 8) | (data[23]! << 16) | (data[24]! << 24); + const width = (raw & 0x3fff) + 1; + const height = ((raw >> 14) & 0x3fff) + 1; + if (width > 0 && height > 0) return { width, height }; + } + } + // JPEG: scan for SOF0/SOF2 marker with dimensions + if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) { + let offset = 2; + while (offset + 8 < data.byteLength) { + if (data[offset] !== 0xff) break; + const marker = data[offset + 1]!; + // Standalone markers (TEM, RSTn, SOI, EOI) carry no length payload. + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { + offset += 2; + continue; + } + const length = (data[offset + 2]! << 8) | data[offset + 3]!; + if (marker === 0xc0 || marker === 0xc2) { + const height = (data[offset + 5]! << 8) | data[offset + 6]!; + const width = (data[offset + 7]! << 8) | data[offset + 8]!; + if (width > 0 && height > 0) return { width, height }; + break; + } + if (length < 2) break; + offset += 2 + length; + } + } + return undefined; +} + +/** + * Build SelectedImage messages for the AgentService vision path: + * store bytes in the local KV map under sha256(blobId), and encode + * `blobIdWithData` so the server can populate its cache without relying solely + * on getBlobArgs timing. Also set `path` like native/shunt clients. + */ +export function buildSelectedImages( + images: readonly ResolvedCursorImage[], + requestScope?: CursorBlobRequestScopeToken, +): SelectedImage[] { + return images.map(image => { + const blobId = storeCursorBlob(image.data, requestScope); + const dims = sniffCursorImageDimensions(image.data); + return create(SelectedImageSchema, { + uuid: image.uuid, + path: cursorImageAttachmentPath(image.uuid, image.mimeType), + mimeType: image.mimeType, + ...(dims + ? { dimension: create(SelectedImage_DimensionSchema, dims) } + : {}), + dataOrBlobId: { + case: "blobIdWithData", + value: create(SelectedImage_BlobIdWithDataSchema, { + blobId, + data: image.data, + }), + }, + }); + }); +} + +/** + * Always send `UserMessage.selected_context`, even when empty — matches cursor-agent. + * When images are present, they are blobIdWithData refs backed by the request-scoped KV store. + */ +export function buildSelectedContext( + images: readonly ResolvedCursorImage[] = [], + requestScope?: CursorBlobRequestScopeToken, +): SelectedContext { + return create(SelectedContextSchema, { + selectedImages: buildSelectedImages(images, requestScope), + }); +} + +/** + * Trailing toolResult image promotion: parts kept for SelectedImage (newest + * {@link MAX_CURSOR_IMAGES}) and the part keys that were promoted. + * Older overflow parts stay on MCP (not omitted) because they are not promoted. + */ +export function extractTrailingToolResultImagePromotion( + messages: readonly OcxMessage[], +): { + parts: CursorImagePartRef[]; + omittedOlder: number; + promotedCallIds: Set; + promotedPartKeys: Set; + trailingBlockStart: number; +} { + const effective = stripTrailingTransparentDeveloperMessages(messages); + let start = effective.length; + while (start > 0 && effective[start - 1]?.role === "toolResult") start -= 1; + + const entries: Array<{ callId: string; partIndex: number; part: CursorImagePartRef }> = []; + for (const message of effective.slice(start)) { + if (message.role !== "toolResult") continue; + let imagePartIndex = 0; + for (const part of extractCursorImageParts(message.content)) { + entries.push({ callId: message.toolCallId, partIndex: imagePartIndex, part }); + imagePartIndex += 1; + } + } + + const kept = entries.length <= MAX_CURSOR_IMAGES + ? entries + : entries.slice(-MAX_CURSOR_IMAGES); + return { + parts: kept.map(entry => entry.part), + omittedOlder: Math.max(0, entries.length - kept.length), + promotedCallIds: new Set(kept.map(entry => entry.callId)), + promotedPartKeys: new Set(kept.map(entry => cursorVisionImagePartKey(entry.callId, entry.partIndex))), + trailingBlockStart: start, + }; +} + +/** + * Collect image parts from trailing consecutive toolResult messages, + * oldest→newest within the block, capped at {@link MAX_CURSOR_IMAGES} (keep newest). + */ +export function extractTrailingToolResultImageParts( + messages: readonly OcxMessage[], +): { parts: CursorImagePartRef[]; omittedOlder: number } { + const { parts, omittedOlder } = extractTrailingToolResultImagePromotion(messages); + return { parts, omittedOlder }; +} + +function developerMessagePlainText(content: OcxMessage["content"] | string): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map(part => (part.type === "text" ? part.text : "")) + .filter(text => text.length > 0) + .join("\n"); +} + +/** + * Codex Desktop multi-agent guidance injected after tool results is transparent for + * Cursor vision / tool-continuation detection so `view_image` still promotes onto + * SelectedImage. Other developer turns (terminal-guard, web-search nudges, etc.) are not. + */ +export function isTransparentCursorVisionSuffix(message: OcxMessage): boolean { + if (message.role !== "developer") return false; + if (extractCursorImageParts(message.content).length > 0) return false; + const text = developerMessagePlainText(message.content); + return text.includes("") && text.includes(""); +} + +/** Drop trailing transparent developers; returns the same array reference when unchanged. */ +export function stripTrailingTransparentDeveloperMessages( + messages: readonly OcxMessage[], +): readonly OcxMessage[] { + let end = messages.length; + while (end > 0) { + const message = messages[end - 1]; + if (!message || !isTransparentCursorVisionSuffix(message)) break; + end -= 1; + } + return end === messages.length ? messages : messages.slice(0, end); +} + +/** True when, ignoring transparent developer suffix, the active turn is a toolResult. */ +export function cursorIsTrailingToolResultContinuation( + messages: readonly OcxMessage[] | undefined, +): boolean { + if (!messages?.length) return false; + return stripTrailingTransparentDeveloperMessages(messages).at(-1)?.role === "toolResult"; +} + +/** + * Resolve images for the active Cursor turn. + * - Trailing user/developer: attach images (SelectedImage). + * - Trailing toolResult run (e.g. Codex `view_image`): same channel — McpImageContent + * alone is not consumed as vision for external Cursor models like grok-4.5. + * - Trailing non-image developer injections (multi_agent_mode) are stripped first so + * Desktop collab guidance does not hide a view_image promotion. + */ +export async function resolveActiveCursorImages( + messages: readonly OcxMessage[] | undefined, + signal?: AbortSignal, +): Promise { + if (!messages?.length) return []; + + const effective = stripTrailingTransparentDeveloperMessages(messages); + const last = effective.at(-1); + if (last?.role === "toolResult") { + const { parts } = extractTrailingToolResultImageParts(effective); + if (parts.length === 0) return []; + return resolveCursorImageParts(parts, signal); + } + + for (let i = effective.length - 1; i >= 0; i--) { + const message = effective[i]; + if (!message) continue; + if (message.role === "user" || message.role === "developer") { + return resolveCursorImageParts(extractCursorImageParts(message.content), signal); + } + } + return []; +} + +function imageDataUrlFromPrepared(image: ResolvedCursorImage): string { + return `data:${image.mimeType};base64,${Buffer.from(image.data).toString("base64")}`; +} + +/** + * Re-encode a single image URL through {@link prepareCursorImageForWire}. + * HTTPS fetches soft-omit on failure. Omitted images become text (caller replaces the part). + */ +export async function prepareCursorImageDataUrl( + imageUrl: string, + detail?: string, + signal?: AbortSignal, +): Promise<{ status: "ready"; imageUrl: string } | { status: "omitted"; reason: string }> { + try { + const resolved = imageUrl.toLowerCase().startsWith("data:") + ? decodeCursorImageDataUrl(imageUrl) + : imageUrl.toLowerCase().startsWith("https:") + ? await fetchHttpsImageBytes(imageUrl, signal) + : null; + if (!resolved) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + if (resolved.data.byteLength === 0) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + const outcome = await prepareCursorImageForWire({ + data: resolved.data, + mimeType: resolved.mimeType, + uuid: randomUUID(), + ...(detail ? { detail } : {}), + }, signal); + if (outcome.status === "omitted") return outcome; + if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + if ( + imageUrl.toLowerCase().startsWith("data:") + && outcome.image.data === resolved.data + && outcome.image.mimeType === resolved.mimeType + ) { + return { status: "ready", imageUrl }; + } + return { status: "ready", imageUrl: imageDataUrlFromPrepared(outcome.image) }; + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } +} + +async function prepareCursorContentParts( + content: string | readonly OcxContentPart[], + signal?: AbortSignal, +): Promise { + if (typeof content === "string" || !Array.isArray(content)) return content; + let changed = false; + const next: OcxContentPart[] = []; + for (const part of content) { + if (part.type === "image" && typeof part.imageUrl === "string" && part.imageUrl.length > 0) { + throwIfImagePhaseAborted(signal); + const prepared = await prepareCursorImageDataUrl(part.imageUrl, part.detail, signal); + if (prepared.status === "omitted") { + changed = true; + next.push({ type: "text", text: prepared.reason }); + continue; + } + if (prepared.imageUrl !== part.imageUrl) changed = true; + next.push({ ...part, imageUrl: prepared.imageUrl }); + } else { + next.push(part); + } + } + return changed ? next : content; +} + +/** + * First original-message index that still needs image prep for the active vision window. + * Historical messages before this index are left untouched (no HTTPS, no decode). + * Indices map 1:1 into `messages` because transparent developers are only stripped from the end. + */ +export function cursorVisionPrepareStartIndex(messages: readonly OcxMessage[]): number { + const effective = stripTrailingTransparentDeveloperMessages(messages); + if (effective.length === 0) return messages.length; + if (effective.at(-1)?.role === "toolResult") { + let start = effective.length; + while (start > 0 && effective[start - 1]?.role === "toolResult") start -= 1; + return start; + } + for (let i = effective.length - 1; i >= 0; i--) { + const role = effective[i]?.role; + if (role === "user" || role === "developer") return i; + } + return effective.length; +} + +/** + * Rewrite image URLs in the active vision window (last user/developer, or trailing + * toolResult block) through the JPEG soft-cap path before protobuf encode. Historical + * messages are left by reference — encode already omits their MCP pixels. HTTPS failures + * become {@link CURSOR_VISION_IMAGE_OMITTED} text so image-only turns stay userMessageAction. + */ +export async function prepareCursorRawMessages( + messages: readonly OcxMessage[] | undefined, + signal?: AbortSignal, +): Promise { + if (!messages?.length) return messages; + const prepareFrom = cursorVisionPrepareStartIndex(messages); + let changed = false; + const out: OcxMessage[] = []; + for (let i = 0; i < messages.length; i++) { + throwIfImagePhaseAborted(signal); + const message = messages[i]!; + if ( + i >= prepareFrom + && (message.role === "user" || message.role === "developer" || message.role === "toolResult") + ) { + const content = await prepareCursorContentParts(message.content, signal); + if (content !== message.content) { + changed = true; + out.push({ ...message, content } as OcxMessage); + continue; + } + } + out.push(message); + } + return changed ? out : messages; +} diff --git a/src/adapters/cursor/live-transport.ts b/src/adapters/cursor/live-transport.ts index eafb2c9b4..d1468b8e1 100644 --- a/src/adapters/cursor/live-transport.ts +++ b/src/adapters/cursor/live-transport.ts @@ -13,6 +13,13 @@ import { type TranslatorBudget, } from "../../lib/translator-budget"; import { activePromptText, prepareCursorRunRequest } from "./protobuf-request"; +import { + createCursorImagePhaseSignal, + prepareCursorRawMessages, + resolveActiveCursorImages, + cursorIsTrailingToolResultContinuation, +} from "./images"; +import { cursorRequestMessagesFromRaw } from "./request-builder"; import { createCursorContextUsageTracker, createCursorProtobufEventState, @@ -390,7 +397,7 @@ export function finalizeAfterDrain(state: ReturnType>; + try { + rawMessages = await prepareCursorRawMessages(request.rawMessages, imagePhase.signal); + selectedImages = await resolveActiveCursorImages(rawMessages, imagePhase.signal); + } finally { + imagePhase.cancel(); + } + const messages = rawMessages === request.rawMessages + ? request.messages + : cursorRequestMessagesFromRaw(rawMessages); + const preparedRequest = { ...request, messages, rawMessages, selectedImages }; // Build the payload once. The estimate is only worth deriving when there is no // carry-forward to fall back on — with a carry present it would never be used (#373). - const prepared = prepareCursorRunRequest(request, { + const prepared = prepareCursorRunRequest(preparedRequest, { estimateInputTokens: contextUsage.carryForwardTokens === undefined, }); this.blobRequestScope = prepared.blobRequestScope; diff --git a/src/adapters/cursor/protobuf-request.ts b/src/adapters/cursor/protobuf-request.ts index 4ede0a482..7f969c78d 100644 --- a/src/adapters/cursor/protobuf-request.ts +++ b/src/adapters/cursor/protobuf-request.ts @@ -13,6 +13,16 @@ import { storeCursorBlob, type CursorBlobRequestScopeToken, } from "./native-exec"; +import { + buildSelectedContext, + CURSOR_VISION_MCP_IMAGE_OMITTED, + CURSOR_VISION_PROMOTE_NUDGE, + cursorIsTrailingToolResultContinuation, + cursorVisionImagePartKey, + decodeCursorImageDataUrl, + extractTrailingToolResultImagePromotion, + stripTrailingTransparentDeveloperMessages, +} from "./images"; import { estimateTokens } from "../../lib/token-estimate"; import { AgentClientMessageSchema, @@ -26,6 +36,7 @@ import { McpArgsSchema, McpSuccessSchema, McpTextContentSchema, + McpImageContentSchema, McpToolCallSchema, McpToolResultContentItemSchema, McpToolResultSchema, @@ -195,7 +206,7 @@ function rootPromptMessages(request: CursorRunRequest, requestScope: CursorBlobR } const externalModel = isCursorExternalWireModel(request.modelId); - const lastRawIsToolResult = messages.at(-1)?.role === "toolResult"; + const lastRawIsToolResult = cursorIsTrailingToolResultContinuation(messages); const activeUserIndex = lastRawIsToolResult ? -1 : lastActionIndex(messages); for (let i = 0; i < messages.length; i++) { @@ -318,7 +329,7 @@ function contentText(message: OcxMessage): string { .map(part => { if (part.type === "text") return part.text; if (part.type === "thinking") return part.thinking; - if (part.type === "image") return `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`; + if (part.type === "image") return undefined; return undefined; }) .filter((value): value is string => typeof value === "string" && value.length > 0) @@ -328,10 +339,96 @@ function contentText(message: OcxMessage): string { function contentToText(content: OcxToolResultMessage["content"]): string { if (typeof content === "string") return content; return content - .map(part => part.type === "text" ? part.text : `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`) + .map(part => part.type === "text" ? part.text : undefined) + .filter((value): value is string => typeof value === "string" && value.length > 0) .join("\n"); } +function toolResultHasImage(content: OcxToolResultMessage["content"]): boolean { + if (typeof content === "string") return false; + return content.some(part => part.type === "image" + && typeof part.imageUrl === "string" + && part.imageUrl.length > 0); +} + +function syntheticToolCallFromResult( + message: OcxToolResultMessage, +): Extract { + return { + type: "toolCall", + id: message.toolCallId, + name: message.toolName, + ...(message.toolNamespace ? { namespace: message.toolNamespace } : {}), + arguments: {}, + }; +} + +type ToolResultImageOmitOptions = { + /** Omit every image part (historical toolResults outside the trailing promote window). */ + omitAllImages?: boolean; + /** Per-part SelectedImage promotion omit (`callId#index`). */ + omitImagePartKeys?: ReadonlySet; + toolCallId?: string; +}; + +function toolResultContentItems( + content: OcxToolResultMessage["content"], + options?: ToolResultImageOmitOptions, +) { + if (typeof content === "string") { + return [create(McpToolResultContentItemSchema, { + content: { case: "text", value: create(McpTextContentSchema, { text: content }) }, + })]; + } + + let imagePartIndex = 0; + const items = content.flatMap(part => { + if (part.type === "text" && part.text.length > 0) { + return [create(McpToolResultContentItemSchema, { + content: { case: "text", value: create(McpTextContentSchema, { text: part.text }) }, + })]; + } + if (part.type === "image" && typeof part.imageUrl === "string" && part.imageUrl.length > 0) { + const partIndex = imagePartIndex; + imagePartIndex += 1; + const partKey = typeof options?.toolCallId === "string" + ? cursorVisionImagePartKey(options.toolCallId, partIndex) + : undefined; + const omitPart = options?.omitAllImages === true + || (partKey !== undefined && options?.omitImagePartKeys?.has(partKey) === true); + if (omitPart) { + return [create(McpToolResultContentItemSchema, { + content: { case: "text", value: create(McpTextContentSchema, { text: CURSOR_VISION_MCP_IMAGE_OMITTED }) }, + })]; + } + try { + if (!part.imageUrl.toLowerCase().startsWith("data:")) return []; + // Shared decoder enforces MAX_CURSOR_IMAGE_DECODE_BYTES and MIME validation. + const { data, mimeType } = decodeCursorImageDataUrl(part.imageUrl); + if (data.byteLength === 0) return []; + return [create(McpToolResultContentItemSchema, { + content: { + case: "image", + value: create(McpImageContentSchema, { + data, + mimeType, + }), + }, + })]; + } catch { + return []; + } + } + return []; + }); + + return items.length > 0 + ? items + : [create(McpToolResultContentItemSchema, { + content: { case: "text", value: create(McpTextContentSchema, { text: "" }) }, + })]; +} + function toolResultToText(message: OcxToolResultMessage): string { return [ "[tool_result]", @@ -351,10 +448,19 @@ function argBytes(value: unknown): Uint8Array { } } +type ConversationTurnImageOptions = { + /** Per-part SelectedImage promotion omit keys (`callId#index`). */ + omitToolResultImagePartKeys?: ReadonlySet; + /** Drop McpImageContent for toolResults before this rawMessages index. */ + trailingBlockStart?: number; + omitHistoricalMcpImagesOutsideTrailing?: boolean; +}; + function toolCallStep( part: Extract, requestScope: CursorBlobRequestScopeToken, result?: OcxToolResultMessage, + options?: ToolResultImageOmitOptions, ): Uint8Array { const args: Record = {}; for (const [key, value] of Object.entries(part.arguments ?? {})) args[key] = argBytes(value); @@ -373,7 +479,7 @@ function toolCallStep( providerIdentifier: OCX_RESPONSES_TOOL_PROVIDER, args, }), - ...(result ? { result: toolResultPart(result) } : {}), + ...(result ? { result: toolResultPart(result, options) } : {}), }), }, }), @@ -381,20 +487,47 @@ function toolCallStep( })), requestScope); } -function toolResultPart(message: OcxToolResultMessage) { +function toolResultPart( + message: OcxToolResultMessage, + options?: ToolResultImageOmitOptions, +) { return create(McpToolResultSchema, { result: { case: "success", value: create(McpSuccessSchema, { isError: message.isError, - content: [create(McpToolResultContentItemSchema, { - content: { case: "text", value: create(McpTextContentSchema, { text: contentToText(message.content) }) }, - })], + content: toolResultContentItems(message.content, { + omitAllImages: options?.omitAllImages, + omitImagePartKeys: options?.omitImagePartKeys, + toolCallId: message.toolCallId, + }), }), }, }); } +function toolResultImageOmitForMessage( + message: OcxToolResultMessage, + absoluteIndex: number, + options?: ConversationTurnImageOptions, +): ToolResultImageOmitOptions | undefined { + if (!toolResultHasImage(message.content)) return undefined; + const trailingStart = options?.trailingBlockStart; + const outsideTrailing = options?.omitHistoricalMcpImagesOutsideTrailing === true + && typeof trailingStart === "number" + && absoluteIndex < trailingStart; + if (outsideTrailing) { + return { omitAllImages: true, toolCallId: message.toolCallId }; + } + if (options?.omitToolResultImagePartKeys && options.omitToolResultImagePartKeys.size > 0) { + return { + omitImagePartKeys: options.omitToolResultImagePartKeys, + toolCallId: message.toolCallId, + }; + } + return { toolCallId: message.toolCallId }; +} + function assistantStep(part: OcxAssistantContentPart, requestScope: CursorBlobRequestScopeToken): Uint8Array | undefined { if (part.type === "toolCall") return toolCallStep(part, requestScope); if (part.type === "thinking") { @@ -428,19 +561,28 @@ function conversationTurns( request: CursorRunRequest, requestScope: CursorBlobRequestScopeToken, historyMessageStart = 0, + options?: ConversationTurnImageOptions, ): Uint8Array[] { const messages = request.rawMessages; if (!messages?.length) return []; const end = lastActionIndex(messages); const externalModel = isCursorExternalWireModel(request.modelId); - const historyEnd = messages.at(-1)?.role === "toolResult" ? messages.length : Math.max(0, end); + // Transparent developer suffixes stay out of turns; they are not toolResult history. + const historyEnd = cursorIsTrailingToolResultContinuation(messages) + ? stripTrailingTransparentDeveloperMessages(messages).length + : Math.max(0, end); const start = externalModel ? Math.max(0, historyMessageStart) : 0; const turns: Uint8Array[] = []; let current: { userMessage: Uint8Array; steps: Uint8Array[] } | undefined; const pendingToolCalls = new Map>(); const flush = () => { if (!current) return; - for (const part of pendingToolCalls.values()) current.steps.push(toolCallStep(part, requestScope)); + // External workers reject result-less historical mcpToolCall; drop unanswered calls. + if (!externalModel) { + for (const part of pendingToolCalls.values()) { + current.steps.push(toolCallStep(part, requestScope, undefined)); + } + } turns.push(storeCursorBlob(toBinary(ConversationTurnStructureSchema, create(ConversationTurnStructureSchema, { turn: { case: "agentConversationTurn", @@ -451,7 +593,8 @@ function conversationTurns( pendingToolCalls.clear(); }; - for (const message of messages.slice(start, historyEnd)) { + for (let absoluteIndex = start; absoluteIndex < historyEnd; absoluteIndex++) { + const message = messages[absoluteIndex]!; if (message.role === "assistant") { if (!current) continue; for (const part of message.content) { @@ -459,6 +602,10 @@ function conversationTurns( // Working external-model clients replay only assistant text. Native mcpToolCall and // ThinkingMessage structures are Composer state and cause external workers to hydrate // the blobs, reach stepCompleted, then reject the turn with invalid_argument. + if (part.type === "toolCall") { + pendingToolCalls.set(part.id, part); + continue; + } if (part.type === "text" && part.text.length > 0) { current.steps.push(storeCursorBlob(toBinary(ConversationStepSchema, create(ConversationStepSchema, { message: { @@ -480,7 +627,14 @@ function conversationTurns( } if (message.role === "toolResult") { if (!current) continue; + const imageOmit = toolResultImageOmitForMessage(message, absoluteIndex, options); if (externalModel) { + if (toolResultHasImage(message.content)) { + const priorCall = pendingToolCalls.get(message.toolCallId) ?? syntheticToolCallFromResult(message); + current.steps.push(toolCallStep(priorCall, requestScope, message, imageOmit)); + pendingToolCalls.delete(message.toolCallId); + continue; + } const prefix = message.isError ? "[Tool Error]" : "[Tool Result]"; current.steps.push(storeCursorBlob(toBinary(ConversationStepSchema, create(ConversationStepSchema, { message: { @@ -488,11 +642,12 @@ function conversationTurns( value: create(AssistantMessageSchema, { text: `${prefix}\n${contentToText(message.content)}` }), }, })), requestScope)); + pendingToolCalls.delete(message.toolCallId); continue; } const priorCall = pendingToolCalls.get(message.toolCallId); if (priorCall) { - current.steps.push(toolCallStep(priorCall, requestScope, message)); + current.steps.push(toolCallStep(priorCall, requestScope, message, imageOmit)); pendingToolCalls.delete(message.toolCallId); } else { current.steps.push(storeCursorBlob(toBinary(ConversationStepSchema, create(ConversationStepSchema, { @@ -509,6 +664,8 @@ function conversationTurns( userMessage: storeCursorBlob(toBinary(UserMessageSchema, create(UserMessageSchema, { text: contentText(message), messageId: crypto.randomUUID(), + selectedContext: buildSelectedContext([], requestScope), + mode: 1, })), requestScope), steps: [], }; @@ -578,18 +735,29 @@ function buildPreparedCursorRunRequest( ? appendCursorGenericToolUseHint(request.tools, rawText) : rawText; // Tool-result-only turns resume the remembered Cursor conversation with results in history. - const lastRawIsToolResult = request.rawMessages?.at(-1)?.role === "toolResult"; - const actionCase = !lastRawIsToolResult && text.trim().length > 0 + // Exception: image-bearing tool results (Codex view_image) must ride SelectedImage on a + // userMessageAction — McpImageContent inline bytes are not hydrated as vision for grok/etc. + const lastRawIsToolResult = cursorIsTrailingToolResultContinuation(request.rawMessages); + const selectedImages = request.selectedImages ?? []; + const promoteToolResultImages = lastRawIsToolResult && selectedImages.length > 0; + const imagePromotion = extractTrailingToolResultImagePromotion(request.rawMessages ?? []); + // Image-only turns that soft-omit to CURSOR_VISION_IMAGE_OMITTED text must stay userMessageAction. + const actionCase = selectedImages.length > 0 || (!lastRawIsToolResult && text.trim().length > 0) ? "userMessageAction" : "resumeAction"; + const actionText = promoteToolResultImages ? CURSOR_VISION_PROMOTE_NUDGE : text; + const selectedContext = buildSelectedContext(selectedImages, requestScope); const action = create(ConversationActionSchema, { action: actionCase === "userMessageAction" ? { case: "userMessageAction", value: create(UserMessageActionSchema, { userMessage: create(UserMessageSchema, { - text, + text: actionText, messageId: crypto.randomUUID(), + selectedContext, + // OmniRoute / cursor-agent always send mode=1 on UserMessage. + mode: 1, }), requestContext: buildRequestContext(), }), @@ -603,7 +771,11 @@ function buildPreparedCursorRunRequest( }); const rootPromptMessagesState = rootPromptMessages(request, requestScope); const rootPromptMessageIds = rootPromptMessagesState.ids; - const turnIds = conversationTurns(request, requestScope, rootPromptMessagesState.historyMessageStart); + const turnIds = conversationTurns(request, requestScope, rootPromptMessagesState.historyMessageStart, { + omitToolResultImagePartKeys: promoteToolResultImages ? imagePromotion.promotedPartKeys : undefined, + trailingBlockStart: imagePromotion.trailingBlockStart, + omitHistoricalMcpImagesOutsideTrailing: true, + }); // Hoisted out of the mcp_tools spread below so the estimate can read the same // filtered definitions the wire carries. Both helpers are pure. const visibleTools = cursorToolsForActivePrompt(request.tools, rawText, request.toolChoice); @@ -675,7 +847,7 @@ function buildPreparedCursorRunRequest( // the event-state `clientToolNames` use (live-transport.ts). Advertising the raw `request.tools` // here would let mcp_tools expose a tool that the event state does not recognize for a generic // tool-count prompt, so a call to it would be rejected as an unknown Responses tool. - ...(mcpToolDefs.length > 0 ? { mcpTools: create(McpToolsSchema, { mcpTools: mcpToolDefs }) } : {}), + mcpTools: create(McpToolsSchema, { mcpTools: mcpToolDefs }), }); const message = create(AgentClientMessageSchema, { @@ -688,7 +860,7 @@ function buildPreparedCursorRunRequest( // tools the payload dropped — the defect that blocked PR #376. const modelVisibleParts = [ ...rootPromptMessagesState.serialized, - ...(actionCase === "userMessageAction" ? [text] : []), + ...(actionCase === "userMessageAction" ? [actionText] : []), ...mcpToolDefs.map(modelVisibleToolText), ]; return { diff --git a/src/adapters/cursor/request-builder.ts b/src/adapters/cursor/request-builder.ts index b60f49fff..2680d4f19 100644 --- a/src/adapters/cursor/request-builder.ts +++ b/src/adapters/cursor/request-builder.ts @@ -23,6 +23,7 @@ import { isBareCodexShellBridgeTool, } from "./tool-definitions"; import { lookupCursorThreadConversation } from "./thread-continuity"; +import { extractCursorImageUrls } from "./images"; /** Probe-verified Cursor Connect boundaries, with byte headroom for the enclosing field. */ export const CURSOR_TOOL_COUNT_LIMIT = 330; @@ -161,7 +162,7 @@ function contentPartToText(part: OcxContentPart | OcxAssistantContentPart): stri case "thinking": return part.thinking; case "image": - return `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`; + return undefined; case "toolCall": // Cursor does not accept OpenAI Responses assistant tool-call parts as native history here. // Rendering them as visible "[tool_call]" text leaks synthetic protocol markers back into @@ -194,9 +195,18 @@ function requestMessage(message: OcxMessage): CursorRequestMessage | undefined { switch (message.role) { case "user": case "developer": - return { role: message.role, content: contentToText(message.content) }; + { + const content = contentToText(message.content); + if (content.length === 0 && extractCursorImageUrls(message.content).length === 0) { + return undefined; + } + return { role: message.role, content }; + } case "assistant": - return { role: "assistant", content: contentToText(message.content) }; + { + const content = contentToText(message.content); + return content.length > 0 ? { role: "assistant", content } : undefined; + } case "toolResult": return { role: "tool", @@ -205,6 +215,19 @@ function requestMessage(message: OcxMessage): CursorRequestMessage | undefined { } } +/** + * Rebuild the text `messages` channel from prepared `rawMessages` so omission markers + * and JPEG-rewritten tool results stay visible to {@link activePromptText}. + */ +export function cursorRequestMessagesFromRaw( + messages: readonly OcxMessage[] | undefined, +): CursorRequestMessage[] { + if (!messages?.length) return []; + return messages + .map(requestMessage) + .filter((message): message is CursorRequestMessage => !!message); +} + export function generatedCursorConversationId(): string { return `cursor_${crypto.randomUUID().replace(/-/g, "")}`; } @@ -255,9 +278,7 @@ export function createCursorRequest( parsed: OcxParsedRequest, options: CreateCursorRequestOptions = {}, ): CursorRunRequest { - const messages = parsed.context.messages - .map(requestMessage) - .filter((message): message is CursorRequestMessage => !!message && message.content.length > 0); + const messages = cursorRequestMessagesFromRaw(parsed.context.messages); const activeText = [...messages].reverse().find(message => message.role === "user" || message.role === "developer")?.content ?? ""; const visibleTools = cursorToolsForActivePrompt(parsed.context.tools, activeText, parsed.options.toolChoice); const budget = applyCursorToolBudget(visibleTools, parsed.options.toolChoice); diff --git a/src/adapters/cursor/types.ts b/src/adapters/cursor/types.ts index b32026a07..4de1223b1 100644 --- a/src/adapters/cursor/types.ts +++ b/src/adapters/cursor/types.ts @@ -1,6 +1,7 @@ import type { OcxUsage } from "../../types"; import type { OcxMessage, OcxRequestOptions, OcxTool } from "../../types"; import type { CursorRoutingLevel } from "./discovery"; +import type { ResolvedCursorImage } from "./images"; export interface CursorRequestedModelParameter { id: string; @@ -16,7 +17,13 @@ export interface CursorRunRequest { conversationId: string; system: string[]; messages: CursorRequestMessage[]; - rawMessages?: OcxMessage[]; + rawMessages?: readonly OcxMessage[]; + /** + * Images for the active user turn. Encoded as SelectedImage blobId refs under + * UserMessage.selected_context (bytes live in the request-scoped KV store for + * getBlobArgs hydration). History stays text-only. + */ + selectedImages?: readonly ResolvedCursorImage[]; tools?: OcxTool[]; toolChoice?: OcxRequestOptions["toolChoice"]; parallelToolCalls?: boolean; diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 8da2f91f8..fecdd1dbd 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -8,6 +8,7 @@ import { ALIBABA_CODING_BASE_URL_CHOICES, ALIBABA_CODING_INTL_BASE_URL, } from "./base-url-choices"; import { + CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS, cursorModelContextWindows, cursorModelIds, @@ -865,17 +866,16 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ modelContextWindows: cursorModelContextWindows(CURSOR_STATIC_MODELS), modelInputModalities: cursorModelInputModalities(CURSOR_STATIC_MODELS), modelReasoningEfforts: cursorModelReasoningEfforts(CURSOR_STATIC_MODELS), + // Blind Cursor models (Auto routers, Composer, GLM-5.2) go through the vision sidecar; + // multimodal hosts (Claude/Gemini/GPT/Kimi/Grok) take native SelectedImage. Catalog still + // advertises image for noVision members so Codex can attach (sidecar option B). + noVisionModels: [...CURSOR_NO_VISION_MODELS], // Kimi K3 documents `max` as its API default, and its Cursor ladder has no `medium` // rung — so applyReasoningLevels' medium->high->first fallback would settle the catalog // default on `high`, the picker would send `high` explicitly, and the request builder's // no-effort fallback to `kimi-k3-max` would never be reached. Mirrors the other K3 // routes (kimi, kimi-code, opencode-go). modelDefaultReasoningEfforts: { "kimi-k3": "max" }, - // Cursor's wire protocol never forwards image parts (request-builder emits an unsupported- - // content marker), so the vision sidecar covers ALL cursor models regardless of what the - // upstream model could natively do. Live-discovered models outside the static list fall back - // to the same marker until they appear here. - noVisionModels: cursorModelIds(CURSOR_STATIC_MODELS), }, { id: "xai", diff --git a/src/types.ts b/src/types.ts index 9de0b94b0..7288e2e60 100644 --- a/src/types.ts +++ b/src/types.ts @@ -203,13 +203,21 @@ export function resolveToolChoiceWireName(tools: readonly Pick 0 && list.includes(modelId.slice(0, colon)); + if (colon > 0 && list.includes(modelId.slice(0, colon))) return true; + for (const entry of list) { + if (entry.length > 1 && entry.endsWith("*")) { + const prefix = entry.slice(0, -1); + if (prefix.length > 0 && modelId.startsWith(prefix)) return true; + } + } + return false; } export type OcxToolChoice = diff --git a/tests/catalog-vision-sidecar-modalities.test.ts b/tests/catalog-vision-sidecar-modalities.test.ts index 27de9d504..d9a9f8d1e 100644 --- a/tests/catalog-vision-sidecar-modalities.test.ts +++ b/tests/catalog-vision-sidecar-modalities.test.ts @@ -2,10 +2,15 @@ import { afterEach, describe, expect, test } from "bun:test"; import { applyProviderConfigHints, gatherRoutedModels } from "../src/codex/catalog"; import { clearModelCache } from "../src/codex/model-cache"; import type { OcxProviderConfig } from "../src/types"; +import { modelInList } from "../src/types"; import { deriveComboCatalogModel } from "../src/codex/catalog"; +import { CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS } from "../src/adapters/cursor/discovery"; import { PROVIDER_REGISTRY } from "../src/providers/registry"; -import { enrichProviderFromRegistry } from "../src/providers/derive"; +import { enrichProviderFromRegistry, providerConfigSeed } from "../src/providers/derive"; import type { CatalogModel } from "../src/types"; +import { parseRequest } from "../src/responses/parser"; +import { routeModel } from "../src/router"; +import { planVisionSidecar } from "../src/vision"; const base: OcxProviderConfig = { adapter: "openai-chat", @@ -13,6 +18,28 @@ const base: OcxProviderConfig = { noVisionModels: ["glm-5.2"], }; +const openAiSidecarFixture = { + providerName: "openai", + provider: { adapter: "openai-responses" as const, baseUrl: "https://chatgpt.test/v1", authMode: "forward" as const }, + accountMode: "direct" as const, + authContext: { kind: "main" as const, accountId: null }, + headers: new Headers({ authorization: "Bearer chatgpt" }), +}; + +function cursorImageParsed(model: string) { + return parseRequest({ + model, + input: [{ + type: "message", + role: "user", + content: [ + { type: "input_text", text: "What is in this screenshot?" }, + { type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" }, + ], + }], + }); +} + describe("vision-sidecar catalog modalities", () => { test("noVisionModels models advertise image input (sidecar gives them eyes)", () => { const hinted = applyProviderConfigHints("opencode-go", base, { id: "glm-5.2", provider: "opencode-go" }); @@ -230,3 +257,73 @@ describe("vision-capable provider models feed combo modalities", () => { expect(hinted.inputModalities).toEqual(["text", "image"]); }); }); + +describe("Cursor native vs sidecar vision registry", () => { + test("curates noVisionModels for Auto/Composer/GLM while advertising image for all static ids", () => { + const cursor = PROVIDER_REGISTRY.find(entry => entry.id === "cursor"); + expect(cursor?.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + for (const model of ["auto", "composer-1", "composer-2.5", "composer-2.5-fast", "composer-9-future", "glm-5.2"]) { + expect(modelInList(cursor?.noVisionModels, model), `${model} should match noVision`).toBe(true); + } + for (const model of ["auto", "composer-2.5", "glm-5.2", "gpt-5.5", "gemini-3-pro", "grok-4.5", "kimi-k3"]) { + expect(cursor?.modelInputModalities?.[model]).toEqual(["text", "image"]); + } + for (const model of ["gpt-5.5", "gemini-3-pro", "grok-4.5", "kimi-k3"]) { + expect(modelInList(cursor?.noVisionModels, model)).toBe(false); + } + for (const model of CURSOR_STATIC_MODELS) { + expect(cursor?.modelInputModalities?.[model.id]).toEqual(["text", "image"]); + } + }); + + test("Cursor catalog hints advertise image for sidecar-covered Composer", () => { + const cursor = providerConfigSeed(PROVIDER_REGISTRY.find(entry => entry.id === "cursor")!); + const hinted = applyProviderConfigHints("cursor", cursor, { + id: "composer-2.5", + provider: "cursor", + contextWindow: 200_000, + inputModalities: ["text"], + }); + expect(hinted.inputModalities).toEqual(["text", "image"]); + }); + + test("Cursor catalog hints keep native image modalities for Grok", () => { + const cursor = providerConfigSeed(PROVIDER_REGISTRY.find(entry => entry.id === "cursor")!); + const hinted = applyProviderConfigHints("cursor", cursor, { + id: "gpt-5.5", + provider: "cursor", + contextWindow: 272_000, + inputModalities: cursor.modelInputModalities?.["gpt-5.5"], + }); + expect(hinted.inputModalities).toEqual(["text", "image"]); + }); + + test("Cursor Auto/Composer image requests plan the vision sidecar", () => { + const config = { + port: 10100, + defaultProvider: "cursor", + providers: { + cursor: providerConfigSeed(PROVIDER_REGISTRY.find(entry => entry.id === "cursor")!), + }, + }; + for (const model of ["cursor/auto", "cursor/composer-2.5", "cursor/glm-5.2"] as const) { + const route = routeModel(config, model); + const plan = planVisionSidecar(config, route.provider, route.modelId, cursorImageParsed(model), openAiSidecarFixture); + expect(plan, `${model} should plan the vision sidecar`).toBeDefined(); + } + }); + + test("Cursor Grok/GPT image requests skip the vision sidecar", () => { + const config = { + port: 10100, + defaultProvider: "cursor", + providers: { + cursor: providerConfigSeed(PROVIDER_REGISTRY.find(entry => entry.id === "cursor")!), + }, + }; + for (const model of ["cursor/grok-4.5", "cursor/gpt-5.5"] as const) { + const route = routeModel(config, model); + expect(planVisionSidecar(config, route.provider, route.modelId, cursorImageParsed(model), openAiSidecarFixture)).toBeUndefined(); + } + }); +}); diff --git a/tests/cursor-adapter.test.ts b/tests/cursor-adapter.test.ts index 1627d8e60..93ddfe240 100644 --- a/tests/cursor-adapter.test.ts +++ b/tests/cursor-adapter.test.ts @@ -235,6 +235,67 @@ describe("Cursor adapter live transport", () => { expect(events.filter(event => event.type === "error")).toHaveLength(1); }); + test("does not retry invalid_argument when multi_agent developer trails toolResult", async () => { + const seen: string[] = []; + let attempts = 0; + const adapter = createCursorAdapter({ + ...provider, + apiKey: "cursor-token", + }, { + createTransport: () => ({ + async *run(request) { + attempts += 1; + seen.push(request.conversationId); + throw Object.assign( + new Error("Cursor invalid request: Cursor Connect error invalid_argument: Error"), + { code: "invalid_argument" }, + ); + }, + writeClient() {}, + }), + }); + + const events: AdapterEvent[] = []; + const body: OcxParsedRequest = { + modelId: "cursor/gpt-5.6-sol", + context: { + messages: [ + { role: "user", content: "read a file", timestamp: 1 }, + { + role: "assistant", + model: "cursor/gpt-5.6-sol", + timestamp: 2, + content: [{ type: "toolCall", id: "call_1", name: "read_file", namespace: "mcp__fs", arguments: { path: "a.txt" } }], + }, + { + role: "toolResult", + toolCallId: "call_1", + toolName: "read_file", + toolNamespace: "mcp__fs", + content: "FILE CONTENTS HERE", + isError: false, + timestamp: 3, + }, + { + role: "developer", + content: "Preferred sub-agent: model \"cursor/gpt-5.6-sol\"", + timestamp: 4, + }, + ], + }, + stream: false, + options: { reasoning: "xhigh" }, + _cursorConversationId: "cursor_corrupt_desktop", + }; + + await adapter.runTurn?.(body, { headers: new Headers() }, event => events.push(event)); + + expect(attempts).toBe(1); + expect(seen).toEqual(["cursor_corrupt_desktop"]); + expect(body._cursorConversationId).toBe("cursor_corrupt_desktop"); + expect(events.filter(event => event.type === "error")).toHaveLength(1); + }); + test("retries external-model invalid_argument on plain-user continuations too", async () => { const seen: string[] = []; let attempts = 0; diff --git a/tests/cursor-blob.test.ts b/tests/cursor-blob.test.ts index 8e0adaa28..71bdd21a5 100644 --- a/tests/cursor-blob.test.ts +++ b/tests/cursor-blob.test.ts @@ -28,6 +28,12 @@ import { encodeCursorRunRequest, prepareCursorRunRequest, } from "../src/adapters/cursor/protobuf-request"; +import { + CURSOR_VISION_MCP_IMAGE_OMITTED, + CURSOR_VISION_PROMOTE_NUDGE, + MAX_CURSOR_IMAGE_DECODE_BYTES, + MAX_CURSOR_IMAGES, +} from "../src/adapters/cursor/images"; import { estimateTokens } from "../src/lib/token-estimate"; import { AgentClientMessageSchema, @@ -36,6 +42,7 @@ import { GetBlobArgsSchema, KvServerMessageSchema, SetBlobArgsSchema, + UserMessageSchema, } from "../src/adapters/cursor/gen/agent_pb"; beforeEach(() => { @@ -47,6 +54,14 @@ afterEach(() => { resetAppOwnedMemoryForTests(); }); +/** Minimal valid 1×1 PNG for SelectedImage / MCP fixtures (not signature-only). */ +const PNG_1X1 = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ), +); + function sha256(data: Uint8Array): Uint8Array { return new Uint8Array(createHash("sha256").update(data).digest()); } @@ -114,6 +129,37 @@ function actionText(bytes: Uint8Array): string | undefined { return action?.case === "userMessageAction" ? action.value.userMessage?.text : undefined; } +function activeUserMessage(bytes: Uint8Array) { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + return action?.case === "userMessageAction" ? action.value.userMessage : undefined; +} + +function activeSelectedImages(bytes: Uint8Array) { + return activeUserMessage(bytes)?.selectedContext?.selectedImages; +} + +function conversationToolCallSteps(bytes: Uint8Array) { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + return (run?.conversationState?.turns ?? []).flatMap(turnId => { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") return []; + return turn.turn.value.steps.map(stepId => fromBinary(ConversationStepSchema, blobData(stepId))); + }).filter(step => step.message.case === "toolCall"); +} + +function nativeTurnUserMessages(bytes: Uint8Array) { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + return (run?.conversationState?.turns ?? []).map(turnId => { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") return undefined; + return fromBinary(UserMessageSchema, blobData(turn.turn.value.userMessage)); + }).filter((message): message is NonNullable => message !== undefined); +} + /** The `toolName`s advertised in the top-level AgentRunRequest.mcp_tools channel (undefined when unset). */ function mcpToolNames(bytes: Uint8Array): string[] | undefined { const msg = fromBinary(AgentClientMessageSchema, bytes); @@ -155,6 +201,185 @@ describe("Cursor blob handshake", () => { expect(run?.mcpTools?.mcpTools[0]?.toolName).toBe("mcp__fs__read_file"); }); + test("encodeCursorRunRequest attaches selectedContext with blobId image refs on the active user turn", () => { + // Native cursor-agent converts SelectedImage.data → sha256 blobId and serves + // bytes via getBlobArgs. Inline data on the run request is ignored for vision. + const imageBytes = PNG_1X1; + const expectedBlobId = new Uint8Array(createHash("sha256").update(imageBytes).digest()); + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "see this" }], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "img-uuid-1", + }], + }); + + expect(actionText(bytes)).toBe("see this"); + const userMessage = activeUserMessage(bytes); + expect(userMessage?.mode).toBe(1); + expect(userMessage?.selectedContext).toBeDefined(); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("img-uuid-1"); + expect(images?.[0]?.mimeType).toBe("image/png"); + expect(images?.[0]?.path).toBe("attachment-img-uuid-1.png"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + const withData = images?.[0]?.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(withData.blobId)).toEqual(Array.from(expectedBlobId)); + expect(Array.from(withData.data)).toEqual(Array.from(imageBytes)); + expect(Array.from(blobData(expectedBlobId))).toEqual(Array.from(imageBytes)); + }); + + test("encodeCursorRunRequest always sends empty selectedContext and mode=1 on text-only turns", () => { + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "hi" }], + }); + + const userMessage = activeUserMessage(bytes); + expect(userMessage?.text).toBe("hi"); + expect(userMessage?.mode).toBe(1); + expect(userMessage?.selectedContext).toBeDefined(); + expect(userMessage?.selectedContext?.selectedImages.length).toBe(0); + }); + + test("encodeCursorRunRequest keeps selectedContext only on the active user turn", () => { + const activeImageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "user", content: "active turn" }], + rawMessages: [ + { + role: "user", + content: [ + { type: "text", text: "old turn" }, + { type: "image", imageUrl: "data:image/png;base64,old", detail: "auto" }, + ], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { role: "user", content: "active turn", timestamp: 3 }, + ], + selectedImages: [{ + data: activeImageBytes, + mimeType: "image/png", + uuid: "active-img", + }], + }); + + const roots = decodeRootMessages(bytes) as Array<{ role?: string; selectedContext?: unknown }>; + expect(roots.some(root => root.selectedContext !== undefined)).toBe(false); + + const historicalUser = nativeTurnUserMessages(bytes)[0]; + expect(historicalUser?.text).toBe("old turn"); + expect(historicalUser?.mode).toBe(1); + expect(historicalUser?.selectedContext).toBeDefined(); + expect(historicalUser?.selectedContext?.selectedImages.length).toBe(0); + + const activeMessage = activeUserMessage(bytes); + expect(activeMessage?.mode).toBe(1); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("active-img"); + expect(images?.[0]?.path).toBe("attachment-active-img.png"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + const activeWithData = images?.[0]?.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(blobData(activeWithData.blobId))).toEqual(Array.from(activeImageBytes)); + expect(Array.from(activeWithData.data)).toEqual(Array.from(activeImageBytes)); + }); + + test("encodeCursorRunRequest uses userMessageAction for image-only turns with selectedImages", () => { + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "" }], + rawMessages: [{ + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 1, + }], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "image-only", + }], + }); + + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + expect(actionText(bytes)).toBe(""); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("image-only"); + expect(images?.[0]?.path).toBe("attachment-image-only.png"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + const imageOnlyWithData = images?.[0]?.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(blobData(imageOnlyWithData.blobId))).toEqual(Array.from(imageBytes)); + expect(Array.from(imageOnlyWithData.data)).toEqual(Array.from(imageBytes)); + }); + + test("encodeCursorRunRequest uses userMessageAction for image-only turns after assistant reply", () => { + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: [], + messages: [ + { role: "user", content: "first" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "" }, + ], + rawMessages: [ + { role: "user", content: "first", timestamp: 1 }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 3, + }, + ], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "follow-up-image", + }], + }); + + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + expect(actionText(bytes)).toBe(""); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("follow-up-image"); + expect(images?.[0]?.path).toBe("attachment-follow-up-image.png"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + const followUpWithData = images?.[0]?.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(blobData(followUpWithData.blobId))).toEqual(Array.from(imageBytes)); + expect(Array.from(followUpWithData.data)).toEqual(Array.from(imageBytes)); + }); + test("caps external root replay while preserving system and newest history", () => { const rawMessages = Array.from({ length: 210 }, (_, index) => index % 2 === 0 @@ -619,6 +844,550 @@ describe("Cursor blob handshake", () => { expect(run?.action?.action.case).toBe("resumeAction"); }); + + test("forwards view_image tool-result data URLs as McpImageContent", () => { + const imageBytes = PNG_1X1; + const imageUrl = `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + rawMessages: [ + { role: "user", content: "describe the image", timestamp: 1 }, + { + role: "assistant", + model: "cursor/composer-2.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [ + { type: "text", text: "image loaded" }, + { type: "image", imageUrl, detail: "auto" }, + ], + isError: false, + timestamp: 3, + }, + ], + }); + + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall") throw new Error("expected toolCall"); + expect(tool.value.tool.case).toBe("mcpToolCall"); + if (tool.value.tool.case !== "mcpToolCall") throw new Error("expected mcpToolCall"); + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + const content = result.result.value.content; + expect(content.length).toBe(2); + expect(content[0]?.content.case).toBe("text"); + expect(content[1]?.content.case).toBe("image"); + if (content[1]?.content.case !== "image") throw new Error("expected image"); + expect(content[1].content.value.mimeType).toBe("image/png"); + expect(Array.from(content[1].content.value.data)).toEqual(Array.from(imageBytes)); + }); + + test("omits oversize tool-result data URLs from McpImageContent via shared decoder", () => { + resetCursorBlobStateForTests(); + const huge = Buffer.alloc(MAX_CURSOR_IMAGE_DECODE_BYTES + 1024, 0x41); + const imageUrl = `data:image/png;base64,${huge.toString("base64")}`; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + rawMessages: [ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/composer-2.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [ + { type: "text", text: "image loaded" }, + { type: "image", imageUrl, detail: "auto" }, + ], + isError: false, + timestamp: 3, + }, + ], + }); + + const toolCalls = conversationToolCallSteps(bytes); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall") throw new Error("expected toolCall"); + expect(tool.value.tool.case).toBe("mcpToolCall"); + if (tool.value.tool.case !== "mcpToolCall") throw new Error("expected mcpToolCall"); + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + const content = result.result.value.content; + expect(content.every(item => item.content.case !== "image")).toBe(true); + expect(content.some(item => item.content.case === "text" && item.content.value.text === "image loaded")).toBe(true); + }); + + test("forwards grok-4.5 view_image tool-result data URLs as McpImageContent", () => { + const imageBytes = PNG_1X1; + const imageUrl = `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + rawMessages: [ + { role: "user", content: "describe the image", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [ + { type: "text", text: "image loaded" }, + { type: "image", imageUrl, detail: "auto" }, + ], + isError: false, + timestamp: 3, + }, + ], + }); + + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall") throw new Error("expected toolCall"); + expect(tool.value.tool.case).toBe("mcpToolCall"); + if (tool.value.tool.case !== "mcpToolCall") throw new Error("expected mcpToolCall"); + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + const content = result.result.value.content; + expect(content.length).toBe(2); + expect(content[0]?.content.case).toBe("text"); + expect(content[1]?.content.case).toBe("image"); + if (content[1]?.content.case !== "image") throw new Error("expected image"); + expect(content[1].content.value.mimeType).toBe("image/png"); + expect(Array.from(content[1].content.value.data)).toEqual(Array.from(imageBytes)); + + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const turn = fromBinary(ConversationTurnStructureSchema, blobData(run?.conversationState?.turns[0] ?? new Uint8Array())); + expect(turn.turn.case).toBe("agentConversationTurn"); + const steps = turn.turn.case === "agentConversationTurn" ? turn.turn.value.steps : []; + expect(steps).toHaveLength(1); + const step = fromBinary(ConversationStepSchema, blobData(steps[0] ?? new Uint8Array())); + expect(step.message.case).toBe("toolCall"); + expect(step.message.case === "toolCall" && step.message.value.tool.case).toBe("mcpToolCall"); + expect(run?.action?.action.case).toBe("resumeAction"); + }); + + test("promotes grok-4.5 view_image images onto userMessageAction SelectedImage", () => { + resetCursorBlobStateForTests(); + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + selectedImages: [{ uuid: "from-view", mimeType: "image/png", data: imageBytes }], + rawMessages: [ + { role: "user", content: "describe the image", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl: `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`, detail: "auto" }], + isError: false, + timestamp: 3, + }, + ], + }); + + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("from-view"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + expect(actionText(bytes)).toBe(CURSOR_VISION_PROMOTE_NUDGE); + + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall") throw new Error("expected toolCall"); + expect(tool.value.tool.case).toBe("mcpToolCall"); + if (tool.value.tool.case !== "mcpToolCall") throw new Error("expected mcpToolCall"); + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + const content = result.result.value.content; + expect(content.every(item => item.content.case !== "image")).toBe(true); + expect(content.some(item => + item.content.case === "text" && item.content.value.text === CURSOR_VISION_MCP_IMAGE_OMITTED + )).toBe(true); + }); + + test("promotes view_image SelectedImage when multi_agent developer trails the toolResult", () => { + // Mirrors Desktop: injectDeveloperMessage appends after view_image output. + resetCursorBlobStateForTests(); + const imageBytes = PNG_1X1; + const collab = "Preferred sub-agent: model \"cursor/grok-4.5\", reasoning_effort \"high\""; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + selectedImages: [{ uuid: "from-view", mimeType: "image/png", data: imageBytes }], + rawMessages: [ + { role: "user", content: "What is in this image? Do not guess.", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl: `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`, detail: "high" }], + isError: false, + timestamp: 3, + }, + { role: "developer", content: collab, timestamp: 4 }, + ], + }); + + expect(activeSelectedImages(bytes)?.length).toBe(1); + expect(actionText(bytes)).toBe(CURSOR_VISION_PROMOTE_NUDGE); + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall") throw new Error("expected toolCall"); + expect(tool.value.tool.case).toBe("mcpToolCall"); + if (tool.value.tool.case !== "mcpToolCall") throw new Error("expected mcpToolCall"); + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + expect(result.result.value.content.every(item => item.content.case !== "image")).toBe(true); + expect(result.result.value.content.some(item => + item.content.case === "text" && item.content.value.text === CURSOR_VISION_MCP_IMAGE_OMITTED + )).toBe(true); + }); + + test("image-only user attach keeps empty action text", () => { + resetCursorBlobStateForTests(); + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "user", content: "" }], + selectedImages: [{ uuid: "attach", mimeType: "image/png", data: imageBytes }], + rawMessages: [ + { + role: "user", + content: [{ type: "image", imageUrl: `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}` }], + timestamp: 1, + }, + ], + }); + expect(activeSelectedImages(bytes)?.length).toBe(1); + expect(actionText(bytes)).toBe(""); + }); + + test("promotes two trailing view_image toolResults onto SelectedImage", () => { + resetCursorBlobStateForTests(); + const a = PNG_1X1; + const b = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]" }], + selectedImages: [ + { uuid: "from-a", mimeType: "image/png", data: a }, + { uuid: "from-b", mimeType: "image/png", data: b }, + ], + rawMessages: [ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [ + { type: "toolCall", id: "call_a", name: "view_image", arguments: { path: "/tmp/a.png" } }, + { type: "toolCall", id: "call_b", name: "view_image", arguments: { path: "/tmp/b.png" } }, + ], + }, + { + role: "toolResult", + toolCallId: "call_a", + toolName: "view_image", + content: [{ type: "image", imageUrl: `data:image/png;base64,${Buffer.from(a).toString("base64")}` }], + isError: false, + timestamp: 3, + }, + { + role: "toolResult", + toolCallId: "call_b", + toolName: "view_image", + content: [{ type: "image", imageUrl: `data:image/png;base64,${Buffer.from(b).toString("base64")}` }], + isError: false, + timestamp: 4, + }, + ], + }); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(2); + expect(actionText(bytes)).toBe(CURSOR_VISION_PROMOTE_NUDGE); + }); + + test("MCP omit on promote only hits call ids that were SelectedImage-promoted", () => { + resetCursorBlobStateForTests(); + // 13 trailing image toolResults; SelectedImage only carries the newest 12 uuids. + // The oldest call must keep McpImageContent (not text-omitted). + const stub = (_n: number) => PNG_1X1; + const toolCalls = Array.from({ length: 13 }, (_, i) => ({ + type: "toolCall" as const, + id: `call_${i}`, + name: "view_image", + arguments: { path: `/tmp/${i}.png` }, + })); + const toolResults = Array.from({ length: 13 }, (_, i) => ({ + role: "toolResult" as const, + toolCallId: `call_${i}`, + toolName: "view_image", + content: [{ + type: "image" as const, + imageUrl: `data:image/png;base64,${Buffer.from(stub(i)).toString("base64")}`, + }], + isError: false, + timestamp: i + 3, + })); + const selectedImages = Array.from({ length: 12 }, (_, i) => ({ + uuid: `keep-${i + 1}`, + mimeType: "image/png", + data: stub(i + 1), + })); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]" }], + selectedImages, + rawMessages: [ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: toolCalls, + }, + ...toolResults, + ], + }); + + expect(activeSelectedImages(bytes)?.length).toBe(12); + const toolSteps = conversationToolCallSteps(bytes); + expect(toolSteps.length).toBe(13); + + const byCallId = new Map[number]>(); + for (const step of toolSteps) { + if (step.message.case !== "toolCall") continue; + if (step.message.value.tool.case !== "mcpToolCall") continue; + byCallId.set(step.message.value.tool.value.args?.toolCallId ?? "", step); + } + + const oldest = byCallId.get("call_0"); + expect(oldest).toBeDefined(); + if (oldest?.message.case !== "toolCall" || oldest.message.value.tool.case !== "mcpToolCall") { + throw new Error("expected mcp toolCall"); + } + const oldestResult = oldest.message.value.tool.value.result; + expect(oldestResult?.result.case).toBe("success"); + if (oldestResult?.result.case !== "success") throw new Error("expected success"); + expect(oldestResult.result.value.content.some(item => item.content.case === "image")).toBe(true); + + const newest = byCallId.get("call_12"); + expect(newest).toBeDefined(); + if (newest?.message.case !== "toolCall" || newest.message.value.tool.case !== "mcpToolCall") { + throw new Error("expected mcp toolCall"); + } + const newestResult = newest.message.value.tool.value.result; + expect(newestResult?.result.case).toBe("success"); + if (newestResult?.result.case !== "success") throw new Error("expected success"); + expect(newestResult.result.value.content.every(item => item.content.case !== "image")).toBe(true); + expect(newestResult.result.value.content.some(item => + item.content.case === "text" && item.content.value.text === CURSOR_VISION_MCP_IMAGE_OMITTED + )).toBe(true); + }); + + test("one toolResult with >12 images promotes 12 and keeps overflow on MCP", () => { + resetCursorBlobStateForTests(); + const imageUrl = `data:image/png;base64,${Buffer.from(PNG_1X1).toString("base64")}`; + const images = Array.from({ length: MAX_CURSOR_IMAGES + 2 }, () => ({ + type: "image" as const, + imageUrl, + })); + const selectedImages = Array.from({ length: MAX_CURSOR_IMAGES }, (_, i) => ({ + uuid: `p-${i}`, + mimeType: "image/png", + data: PNG_1X1, + })); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]" }], + selectedImages, + rawMessages: [ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_many", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_many", + toolName: "view_image", + content: images, + isError: false, + timestamp: 3, + }, + ], + }); + + expect(activeSelectedImages(bytes)?.length).toBe(MAX_CURSOR_IMAGES); + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall" || tool.value.tool.case !== "mcpToolCall") { + throw new Error("expected mcpToolCall"); + } + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + const content = result.result.value.content; + const mcpImages = content.filter(item => item.content.case === "image"); + const omitted = content.filter(item => + item.content.case === "text" && item.content.value.text === CURSOR_VISION_MCP_IMAGE_OMITTED + ); + // Oldest 2 overflow stay on MCP; newest 12 are SelectedImage-omitted. + expect(mcpImages.length).toBe(2); + expect(omitted.length).toBe(MAX_CURSOR_IMAGES); + }); + + test("prior view_image then new user turn drops historical McpImageContent", () => { + resetCursorBlobStateForTests(); + const imageUrl = `data:image/png;base64,${Buffer.from(PNG_1X1).toString("base64")}`; + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "user", content: "thanks" }], + rawMessages: [ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl }], + isError: false, + timestamp: 3, + }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 4, + content: [{ type: "text", text: "a cat" }], + }, + { role: "user", content: "thanks", timestamp: 5 }, + ], + }); + + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.length).toBe(1); + const tool = toolCalls[0]?.message; + expect(tool?.case).toBe("toolCall"); + if (tool?.case !== "toolCall" || tool.value.tool.case !== "mcpToolCall") { + throw new Error("expected mcpToolCall"); + } + const result = tool.value.tool.value.result; + expect(result?.result.case).toBe("success"); + if (result?.result.case !== "success") throw new Error("expected success"); + expect(result.result.value.content.every(item => item.content.case !== "image")).toBe(true); + expect(result.result.value.content.some(item => + item.content.case === "text" && item.content.value.text === CURSOR_VISION_MCP_IMAGE_OMITTED + )).toBe(true); + }); + + test("external assistant toolCall without result is dropped from historical turns", () => { + resetCursorBlobStateForTests(); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "user", content: "continue" }], + rawMessages: [ + { role: "user", content: "start", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [ + { type: "text", text: "calling" }, + { type: "toolCall", id: "call_orphan", name: "read_file", arguments: { path: "/tmp/x" } }, + ], + }, + { role: "user", content: "continue", timestamp: 3 }, + ], + }); + + const toolCalls = conversationToolCallSteps(bytes); + expect(toolCalls.every(step => { + if (step.message.case !== "toolCall") return true; + if (step.message.value.tool.case !== "mcpToolCall") return true; + return step.message.value.tool.value.result !== undefined; + })).toBe(true); + expect(toolCalls.length).toBe(0); + }); }); describe("Cursor AgentRunRequest.mcp_tools channel", () => { @@ -650,7 +1419,7 @@ describe("Cursor AgentRunRequest.mcp_tools channel", () => { expect(mcpToolNames(bytes)).toEqual(["exec_command"]); }); - test("leaves mcp_tools unset when tools are empty", () => { + test("sends empty mcp_tools wrapper when tools are empty", () => { const bytes = encodeCursorRunRequest({ modelId: "gpt-5.6-luna-high", conversationId: "c1", @@ -658,10 +1427,14 @@ describe("Cursor AgentRunRequest.mcp_tools channel", () => { messages: [{ role: "user", content: "hi" }], tools: [], }); - expect(mcpToolNames(bytes)).toBeUndefined(); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.mcpTools).toBeDefined(); + expect(run?.mcpTools?.mcpTools.length).toBe(0); + expect(mcpToolNames(bytes)).toEqual([]); }); - test("leaves mcp_tools unset when toolChoice is none", () => { + test("sends empty mcp_tools wrapper when toolChoice is none", () => { const bytes = encodeCursorRunRequest({ modelId: "gpt-5.6-luna-high", conversationId: "c1", @@ -670,7 +1443,11 @@ describe("Cursor AgentRunRequest.mcp_tools channel", () => { toolChoice: "none", tools: [{ name: "js", namespace: "mcp__node_repl", description: "Run JS", parameters: {} }], }); - expect(mcpToolNames(bytes)).toBeUndefined(); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.mcpTools).toBeDefined(); + expect(run?.mcpTools?.mcpTools.length).toBe(0); + expect(mcpToolNames(bytes)).toEqual([]); }); }); diff --git a/tests/cursor-discovery.test.ts b/tests/cursor-discovery.test.ts index c8343deda..94e1efcfb 100644 --- a/tests/cursor-discovery.test.ts +++ b/tests/cursor-discovery.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { CURSOR_AUTO_WIRE_MODEL_ID, CURSOR_DEFAULT_CONTEXT_WINDOW, + CURSOR_NO_VISION_MODELS, CURSOR_ROUTER_MODEL_IDS, CURSOR_ROUTING_LEVELS, CURSOR_STATIC_MODELS, @@ -133,6 +134,25 @@ describe("Cursor discovery metadata", () => { expect(inferCursorContextWindow("gpt-5.5")).toBe(272_000); }); + test("no-vision list is a curated subset of the static seed", () => { + const ids = new Set(cursorModelIds(CURSOR_STATIC_MODELS)); + expect([...CURSOR_NO_VISION_MODELS]).toEqual([ + ...CURSOR_ROUTER_MODEL_IDS, + "composer-*", + "glm-5.2", + ]); + for (const id of CURSOR_NO_VISION_MODELS) { + if (id.endsWith("*")) continue; + expect(ids.has(id), `${id} must be in the static Cursor seed`).toBe(true); + } + for (const id of ["composer-1", "composer-2.5", "composer-2.5-fast"]) { + expect(ids.has(id)).toBe(true); + } + for (const id of ["grok-4.5", "grok-4.5-fast", "gpt-5.5", "claude-sonnet-5", "kimi-k3", "gemini-3-pro"]) { + expect(CURSOR_NO_VISION_MODELS as readonly string[]).not.toContain(id); + } + }); + test("input modalities are cloned per model", () => { const modalities = cursorModelInputModalities([{ id: "auto" }]); diff --git a/tests/cursor-effort-suffix.test.ts b/tests/cursor-effort-suffix.test.ts index 2d4c627f0..e9317c8ff 100644 --- a/tests/cursor-effort-suffix.test.ts +++ b/tests/cursor-effort-suffix.test.ts @@ -93,11 +93,23 @@ describe("Cursor per-model reasoning-effort suffix", () => { }); test("grok-4.5 uses current tiers and sends Fast as a separate model parameter", () => { + // Regular Grok wire ids keep Cursor's live-discovery `cursor-` prefix (#1159/#1208). expect(modelIdFor("cursor/grok-4.5", "low")).toBe("cursor-grok-4.5-low"); expect(modelIdFor("cursor/grok-4.5", "medium")).toBe("cursor-grok-4.5-medium"); expect(modelIdFor("cursor/grok-4.5", "high")).toBe("cursor-grok-4.5-high"); expect(modelIdFor("cursor/grok-4.5", "xhigh")).toBe("cursor-grok-4.5-high"); expect(modelIdFor("cursor/grok-4.5")).toBe("cursor-grok-4.5-high"); + // Cursor Start / some plans reject -low; none/minimal clamp to medium. + expect(modelIdFor("cursor/grok-4.5", "none")).toBe("cursor-grok-4.5-medium"); + expect(modelIdFor("cursor/grok-4.5", "minimal")).toBe("cursor-grok-4.5-medium"); + expect(selectionFor("cursor/grok-4.5", "none")).toEqual({ + modelId: "cursor-grok-4.5-medium", + parameters: undefined, + }); + expect(selectionFor("cursor/grok-4.5-fast", "none")).toEqual({ + modelId: "grok-4.5", + parameters: [{ id: "effort", value: "medium" }, { id: "fast", value: "true" }], + }); expect(selectionFor("cursor/grok-4.5", "high")).toEqual({ modelId: "cursor-grok-4.5-high", parameters: undefined, diff --git a/tests/cursor-images.test.ts b/tests/cursor-images.test.ts new file mode 100644 index 000000000..a18d306ec --- /dev/null +++ b/tests/cursor-images.test.ts @@ -0,0 +1,822 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { create, fromBinary } from "@bufbuild/protobuf"; +import { + CursorImageError, + CURSOR_VISION_IMAGE_OMITTED, + CURSOR_VISION_PROMOTE_NUDGE, + CURSOR_VISION_SOFT_MAX_BYTES, + CURSOR_VISION_SOFT_MAX_BYTES_HIGH, + MAX_CURSOR_IMAGE_BYTES, + MAX_CURSOR_IMAGE_DECODE_BYTES, + MAX_CURSOR_IMAGE_DECODE_EDGE, + MAX_CURSOR_IMAGE_PIXELS, + MAX_CURSOR_IMAGES, + buildSelectedImages, + createCursorImagePhaseSignal, + cursorVisionPrepareStartIndex, + decodeCursorImageDataUrl, + extractTrailingToolResultImageParts, + extractTrailingToolResultImagePromotion, + prepareCursorImageForWire, + prepareCursorRawMessages, + isTransparentCursorVisionSuffix, + resolveActiveCursorImages, + resolveCursorImages, + sniffCursorImageDimensions, + sniffCursorImageFormat, + stripTrailingTransparentDeveloperMessages, +} from "../src/adapters/cursor/images"; +import { + handleCursorNativeKv, + resetCursorBlobStateForTests, +} from "../src/adapters/cursor/native-exec"; +import { cursorRequestMessagesFromRaw } from "../src/adapters/cursor/request-builder"; +import { activePromptText, encodeCursorRunRequest } from "../src/adapters/cursor/protobuf-request"; +import { + AgentClientMessageSchema, + GetBlobArgsSchema, + KvServerMessageSchema, +} from "../src/adapters/cursor/gen/agent_pb"; +import * as destinationPolicy from "../src/lib/destination-policy"; +import * as imageArtifacts from "../src/images/artifacts"; + +/** Minimal valid 1×1 PNG (real IHDR; not a signature-only stub). */ +const PNG_BYTES = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ), +); +const PNG_DATA_URL = `data:image/png;base64,${Buffer.from(PNG_BYTES).toString("base64")}`; + +const spies: Array> = []; + +function mockSpy( + object: T, + method: K, + implementation: T[K], +): ReturnType { + const spy = spyOn(object, method).mockImplementation(implementation as never); + spies.push(spy); + return spy; +} + +afterEach(() => { + while (spies.length > 0) spies.pop()?.mockRestore(); +}); + +function mockPublicHttpsFetch(body = PNG_BYTES, mimeType = "image/png") { + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "public" as const })); + mockSpy(destinationPolicy, "resolvePublicAddresses", async () => ({ + addresses: [{ address: "93.184.216.34", family: 4 }], + hostname: "example.com", + })); + mockSpy(imageArtifacts, "pinnedHttpsGet", async () => new Response(body, { + status: 200, + headers: { "content-type": mimeType }, + })); +} + +async function oversizedDecodablePng(): Promise { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const src = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + return new Uint8Array(await new Bun.Image(src).resize(2400, 2400).png().bytes()); +} + +describe("Cursor image resolver", () => { + test("rejects more than MAX_CURSOR_IMAGES in one request", async () => { + const urls = Array.from({ length: MAX_CURSOR_IMAGES + 1 }, () => PNG_DATA_URL); + await expect(resolveCursorImages(urls)).rejects.toMatchObject({ + name: "CursorImageError", + message: `Too many images in one request (max ${MAX_CURSOR_IMAGES}).`, + }); + }); + + test("omits data URLs above the inbound decode bomb ceiling", async () => { + const oversized = "A".repeat(Math.ceil((MAX_CURSOR_IMAGE_DECODE_BYTES + 1) * 4 / 3)); + // Soft-omit: one bad URL must not abort a mixed turn. + const resolved = await resolveCursorImages([`data:image/png;base64,${oversized}`]); + expect(resolved).toEqual([]); + }); + + test("prep-before-cap accepts PNG over 1 MiB that JPEG-encodes under the soft and wire caps", async () => { + const png = await oversizedDecodablePng(); + expect(png.byteLength).toBeGreaterThan(MAX_CURSOR_IMAGE_BYTES); + const url = `data:image/png;base64,${Buffer.from(png).toString("base64")}`; + const resolved = await resolveCursorImages([url]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + expect(resolved[0]!.data.byteLength).toBeLessThan(png.byteLength); + expect(resolved[0]!.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(resolved[0]!.data.byteLength).toBeLessThanOrEqual(MAX_CURSOR_IMAGE_BYTES); + }); + + test("omits undecodable payloads under the decode ceiling instead of sending them", async () => { + const junk = "A".repeat(Math.ceil((MAX_CURSOR_IMAGE_BYTES + 1) * 4 / 3)); + // Pad to valid base64 length so alphabet/padding checks pass and Bun decode fails. + const padded = junk + "=".repeat((4 - (junk.length % 4)) % 4); + const resolved = await resolveCursorImages([`data:image/png;base64,${padded}`]); + expect(resolved).toEqual([]); + }); + + test("decodes valid base64 data URLs through JPEG prep", async () => { + const resolved = await resolveCursorImages([PNG_DATA_URL]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + expect(resolved[0]!.data.byteLength).toBeGreaterThan(0); + expect(resolved[0]!.data[0]).toBe(0xff); + expect(resolved[0]!.data[1]).toBe(0xd8); + expect(resolved[0]?.uuid.length).toBeGreaterThan(0); + }); + + test("soft-omits malformed and non-image data URLs", async () => { + expect(await resolveCursorImages(["data:image/png,not-base64"])).toEqual([]); + expect(await resolveCursorImages(["data:text/plain;base64,YQ=="])).toEqual([]); + expect(await resolveCursorImages(["data:image/png;base64"])).toEqual([]); + expect(await resolveCursorImages(["data:image/png;base64,"])).toEqual([]); + }); + + test("soft-omits non-HTTPS remote URLs", async () => { + expect(await resolveCursorImages(["http://example.com/image.png"])).toEqual([]); + }); + + test("soft-omits blocked destinations before DNS resolution", async () => { + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "loopback" as const })); + expect(await resolveCursorImages(["https://127.0.0.1/image.png"])).toEqual([]); + }); + + test("soft-omits remote URLs when public DNS resolution fails", async () => { + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "public" as const })); + mockSpy(destinationPolicy, "resolvePublicAddresses", async () => { + throw new Error("blocked"); + }); + expect(await resolveCursorImages(["https://example.com/image.png"])).toEqual([]); + }); + + test("fetches HTTPS images through pinned HTTPS with image content-type", async () => { + mockPublicHttpsFetch(); + const resolved = await resolveCursorImages(["https://example.com/image.png"]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + expect(resolved[0]!.data[0]).toBe(0xff); + expect(resolved[0]!.data[1]).toBe(0xd8); + }); + + test("soft-omits HTTPS responses without an image content-type", async () => { + mockPublicHttpsFetch(PNG_BYTES, "text/plain"); + expect(await resolveCursorImages(["https://example.com/not-image"])).toEqual([]); + }); + + test("resolveActiveCursorImages selects the last user turn and ignores earlier images", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "user", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/auto", + content: [{ type: "text", text: "seen" }], + timestamp: 2, + }, + { + role: "user", + content: [ + { type: "text", text: "active" }, + { type: "image", imageUrl: PNG_DATA_URL }, + ], + timestamp: 3, + }, + ]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + }); + + test("resolveActiveCursorImages supports developer turns", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "developer", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + ]); + expect(resolved).toHaveLength(1); + }); + + test("resolveActiveCursorImages returns empty for text-only trailing toolResult", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "user", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call-1", + toolName: "read_file", + content: "done", + isError: false, + timestamp: 2, + }, + ]); + expect(resolved).toEqual([]); + }); + + test("resolveActiveCursorImages promotes trailing view_image toolResult images", async () => { + const resolved = await resolveActiveCursorImages([ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl: PNG_DATA_URL, detail: "auto" }], + isError: false, + timestamp: 2, + }, + ]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.data.byteLength).toBeGreaterThan(0); + }); + + test("transparent developer suffix does not block view_image SelectedImage promotion", async () => { + // Desktop injects after toolResult; that must stay transparent for vision. + const collab = "Preferred sub-agent: model \"cursor/grok-4.5\", reasoning_effort \"high\""; + const messages = [ + { role: "user" as const, content: "What is in this image? Do not guess.", timestamp: 1 }, + { + role: "assistant" as const, + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall" as const, id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult" as const, + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image" as const, imageUrl: PNG_DATA_URL, detail: "high" }], + isError: false, + timestamp: 3, + }, + { role: "developer" as const, content: collab, timestamp: 4 }, + ]; + expect(isTransparentCursorVisionSuffix(messages[3]!)).toBe(true); + expect(stripTrailingTransparentDeveloperMessages(messages).at(-1)?.role).toBe("toolResult"); + const resolved = await resolveActiveCursorImages(messages); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.data.byteLength).toBeGreaterThan(0); + }); + + test("non-multi_agent developer after toolResult is not transparent", async () => { + const messages = [ + { role: "user" as const, content: "first", timestamp: 1 }, + { + role: "toolResult" as const, + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image" as const, imageUrl: PNG_DATA_URL }], + isError: false, + timestamp: 2, + }, + { role: "developer" as const, content: "Answer with the forced web-search result only.", timestamp: 3 }, + ]; + expect(isTransparentCursorVisionSuffix(messages[2]!)).toBe(false); + expect(stripTrailingTransparentDeveloperMessages(messages)).toBe(messages); + const resolved = await resolveActiveCursorImages(messages); + expect(resolved).toEqual([]); + }); + + test("user message after toolResult is not transparent (does not promote stale tool images)", async () => { + const resolved = await resolveActiveCursorImages([ + { role: "user", content: "first", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + isError: false, + timestamp: 2, + }, + { role: "user", content: "new question without an image", timestamp: 3 }, + ]); + expect(resolved).toEqual([]); + }); + + test("resolveActiveCursorImages collects consecutive trailing image toolResults", async () => { + const second = `data:image/png;base64,${Buffer.from([...PNG_BYTES, 1]).toString("base64")}`; + const resolved = await resolveActiveCursorImages([ + { role: "user", content: "describe both", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_a", + toolName: "view_image", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + isError: false, + timestamp: 2, + }, + { + role: "toolResult", + toolCallId: "call_b", + toolName: "view_image", + content: [{ type: "image", imageUrl: second }], + isError: false, + timestamp: 3, + }, + ]); + expect(resolved).toHaveLength(2); + }); + + test("extractTrailingToolResultImageParts keeps the newest MAX_CURSOR_IMAGES", () => { + const urls = Array.from({ length: MAX_CURSOR_IMAGES + 2 }, (_, i) => ( + `data:image/png;base64,${Buffer.from([...PNG_BYTES, i]).toString("base64")}` + )); + const messages = urls.map((imageUrl, i) => ({ + role: "toolResult" as const, + toolCallId: `call_${i}`, + toolName: "view_image", + content: [{ type: "image" as const, imageUrl }], + isError: false, + timestamp: i + 1, + })); + const { parts, omittedOlder } = extractTrailingToolResultImageParts(messages); + expect(omittedOlder).toBe(2); + expect(parts).toHaveLength(MAX_CURSOR_IMAGES); + expect(parts[0]?.imageUrl).toBe(urls[2]); + expect(parts.at(-1)?.imageUrl).toBe(urls.at(-1)); + }); + + test("CursorImageError carries HTTP status for callers", () => { + const error = new CursorImageError("blocked", 403); + expect(error.status).toBe(403); + expect(error.name).toBe("CursorImageError"); + }); + + test("buildSelectedImages uses blobIdWithData + attachment path and keeps KV hydrated", () => { + resetCursorBlobStateForTests(); + // Minimal PNG signature + IHDR claiming 2x3 (not Bun-decodable — stays PNG) + const png = Uint8Array.from([ + 137, 80, 78, 71, 13, 10, 26, 10, + 0, 0, 0, 13, 73, 72, 68, 82, + 0, 0, 0, 2, 0, 0, 0, 3, + 8, 2, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(sniffCursorImageDimensions(png)).toEqual({ width: 2, height: 3 }); + + // Standalone RST0 before SOF0 must not be parsed as a length-bearing segment. + const jpegWithRst = Uint8Array.from([ + 0xff, 0xd8, // SOI + 0xff, 0xd0, // RST0 (no length) + 0xff, 0xc0, 0x00, 0x0b, 0x08, 0x00, 0x03, 0x00, 0x02, 0x03, 0x01, 0x11, 0x00, // SOF0 2x3 + ]); + expect(sniffCursorImageDimensions(jpegWithRst)).toEqual({ width: 2, height: 3 }); + + const [selected] = buildSelectedImages([{ + data: png, + mimeType: "image/png", + uuid: "u-dim", + }]); + expect(selected?.dataOrBlobId.case).toBe("blobIdWithData"); + expect(selected?.path).toBe("attachment-u-dim.png"); + expect(selected?.dimension?.width).toBe(2); + expect(selected?.dimension?.height).toBe(3); + const withData = selected!.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(withData.blobId)).toEqual(Array.from(createHash("sha256").update(png).digest())); + expect(Array.from(withData.data)).toEqual(Array.from(png)); + + const reply = fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id: 1, + message: { case: "getBlobArgs", value: create(GetBlobArgsSchema, { blobId: withData.blobId }) }, + }))); + const kv = reply.message.case === "kvClientMessage" ? reply.message.value : undefined; + const data = kv?.message.case === "getBlobResult" ? kv.message.value.blobData : undefined; + expect(Array.from(data ?? [])).toEqual(Array.from(png)); + }); + + test("prepareCursorImageForWire re-encodes large PNG as JPEG under the soft cap", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + expect(png.byteLength).toBeGreaterThan(CURSOR_VISION_SOFT_MAX_BYTES); + + const prepared = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "big-png", + }); + expect(prepared.status).toBe("ready"); + if (prepared.status !== "ready") throw new Error("expected ready"); + expect(prepared.image.mimeType).toBe("image/jpeg"); + expect(prepared.image.data.byteLength).toBeLessThan(png.byteLength); + expect(prepared.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(prepared.image.data[0]).toBe(0xff); + expect(prepared.image.data[1]).toBe(0xd8); + }); + + test("detail original/high uses a higher soft tier than auto", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + const auto = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "auto", + detail: "auto", + }); + const original = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "original", + detail: "original", + }); + expect(auto.status).toBe("ready"); + expect(original.status).toBe("ready"); + if (auto.status !== "ready" || original.status !== "ready") throw new Error("expected ready"); + expect(original.image.data.byteLength).toBeGreaterThan(auto.image.data.byteLength); + expect(auto.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(original.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES_HIGH); + expect(original.image.data.byteLength).toBeLessThanOrEqual(MAX_CURSOR_IMAGE_BYTES); + }); + + test("exotic MIME and corrupt PNG fail closed", async () => { + const bmp = await prepareCursorImageForWire({ + data: new Uint8Array([0x42, 0x4d, 0, 0, 0, 0]), + mimeType: "image/bmp", + uuid: "bmp", + }); + expect(bmp).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + + const corrupt = await prepareCursorImageForWire({ + data: new Uint8Array(128).fill(0x41), + mimeType: "image/png", + uuid: "corrupt", + }); + expect(corrupt).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + + // Soft-cap-sized labeled JPEG must still decode; junk under the soft max is omitted. + const fakeJpeg = await prepareCursorImageForWire({ + data: new Uint8Array(128).fill(0xff), + mimeType: "image/jpeg", + uuid: "fake-jpeg", + }); + expect(fakeJpeg).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("extractTrailingToolResultImagePromotion scopes call ids and part keys to kept parts", () => { + const urls = Array.from({ length: MAX_CURSOR_IMAGES + 2 }, (_, i) => ( + `data:image/png;base64,${Buffer.from([...PNG_BYTES, i]).toString("base64")}` + )); + const messages = urls.map((imageUrl, i) => ({ + role: "toolResult" as const, + toolCallId: `call_${i}`, + toolName: "view_image", + content: [{ type: "image" as const, imageUrl }], + isError: false, + timestamp: i + 1, + })); + const { parts, omittedOlder, promotedCallIds, promotedPartKeys } = + extractTrailingToolResultImagePromotion(messages); + expect(omittedOlder).toBe(2); + expect(parts).toHaveLength(MAX_CURSOR_IMAGES); + expect(promotedCallIds.has("call_0")).toBe(false); + expect(promotedCallIds.has("call_1")).toBe(false); + expect(promotedCallIds.has("call_2")).toBe(true); + expect(promotedCallIds.has(`call_${MAX_CURSOR_IMAGES + 1}`)).toBe(true); + expect(promotedPartKeys.has("call_0#0")).toBe(false); + expect(promotedPartKeys.has("call_2#0")).toBe(true); + expect(promotedPartKeys.size).toBe(MAX_CURSOR_IMAGES); + }); + + test("promote nudge warns against path/filename inference", () => { + expect(CURSOR_VISION_PROMOTE_NUDGE).toContain("Describe the image from the tool result."); + expect(CURSOR_VISION_PROMOTE_NUDGE).toContain("Do not infer content from file paths or names."); + }); + + test("prepareCursorRawMessages JPEG-preps view_image tool-result data URLs", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + const imageUrl = `data:image/png;base64,${Buffer.from(png).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { role: "user", content: "describe", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + timestamp: 2, + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl, detail: "auto" }], + isError: false, + timestamp: 3, + }, + ]); + const tool = prepared?.[2]; + expect(tool?.role).toBe("toolResult"); + if (tool?.role !== "toolResult" || typeof tool.content === "string") throw new Error("expected image parts"); + const part = tool.content.find(item => item.type === "image"); + expect(part?.type).toBe("image"); + if (part?.type !== "image") throw new Error("expected image"); + expect(part.imageUrl.startsWith("data:image/jpeg;base64,")).toBe(true); + const payload = part.imageUrl.slice(part.imageUrl.indexOf(",") + 1); + const bytes = Buffer.from(payload, "base64"); + expect(bytes.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(bytes[0]).toBe(0xff); + expect(bytes[1]).toBe(0xd8); + }); + + test("prepareCursorRawMessages replaces exotic images with omission text", async () => { + const bmpUrl = `data:image/bmp;base64,${Buffer.from([0x42, 0x4d, 0, 0]).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: bmpUrl }], + timestamp: 1, + }, + ]); + const user = prepared?.[0]; + expect(user?.role).toBe("user"); + if (user?.role !== "user" || typeof user.content === "string") throw new Error("expected parts"); + expect(user.content).toEqual([{ type: "text", text: CURSOR_VISION_IMAGE_OMITTED }]); + }); + + test("cursorRequestMessagesFromRaw surfaces omission text after prepare", async () => { + const bmpUrl = `data:image/bmp;base64,${Buffer.from([0x42, 0x4d, 0, 0]).toString("base64")}`; + const raw = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: bmpUrl }], + timestamp: 1, + }, + ]); + const messages = cursorRequestMessagesFromRaw(raw); + expect(messages).toEqual([{ role: "user", content: CURSOR_VISION_IMAGE_OMITTED }]); + expect(activePromptText({ + modelId: "grok-4.5", + conversationId: "cursor_test", + system: [], + messages, + rawMessages: raw, + })).toBe(CURSOR_VISION_IMAGE_OMITTED); + }); + + test("live-transport image phase: prepare rawMessages then resolve SelectedImage", async () => { + // Mirrors live-transport.ts: prepareCursorRawMessages → resolveActiveCursorImages + // without injecting selectedImages into encode. + const collab = "Preferred sub-agent: model \"cursor/grok-4.5\""; + const rawIn = [ + { role: "user" as const, content: "What is in this image?", timestamp: 1 }, + { + role: "assistant" as const, + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall" as const, id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult" as const, + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image" as const, imageUrl: PNG_DATA_URL, detail: "high" }], + isError: false, + timestamp: 3, + }, + { role: "developer" as const, content: collab, timestamp: 4 }, + ]; + const rawMessages = await prepareCursorRawMessages(rawIn); + const messages = cursorRequestMessagesFromRaw(rawMessages); + const selectedImages = await resolveActiveCursorImages(rawMessages); + expect(selectedImages).toHaveLength(1); + + resetCursorBlobStateForTests(); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c-wire", + system: ["You are helpful."], + messages, + rawMessages, + selectedImages, + }); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + if (run?.action?.action.case !== "userMessageAction") throw new Error("expected userMessageAction"); + expect(run.action.action.value.userMessage?.text).toBe(CURSOR_VISION_PROMOTE_NUDGE); + expect(run.action.action.value.userMessage?.selectedContext?.selectedImages.length).toBe(1); + }); + + test("image-only HTTPS soft-omit yields userMessageAction with omission text", async () => { + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "public" as const })); + mockSpy(destinationPolicy, "resolvePublicAddresses", async () => { + throw new Error("DNS failed"); + }); + const raw = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: "https://example.com/missing.png" }], + timestamp: 1, + }, + ]); + const messages = cursorRequestMessagesFromRaw(raw); + expect(messages).toEqual([{ role: "user", content: CURSOR_VISION_IMAGE_OMITTED }]); + const selectedImages = await resolveActiveCursorImages(raw); + expect(selectedImages).toEqual([]); + resetCursorBlobStateForTests(); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c-https-omit", + system: [], + messages, + rawMessages: raw, + selectedImages, + }); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + expect(actionTextFrom(bytes)).toBe(CURSOR_VISION_IMAGE_OMITTED); + }); + + test("failed HTTPS with valid text continues text-only", async () => { + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "public" as const })); + mockSpy(destinationPolicy, "resolvePublicAddresses", async () => { + throw new Error("DNS failed"); + }); + const raw = await prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "text", text: "what color is the sky?" }, + { type: "image", imageUrl: "https://example.com/missing.png" }, + ], + timestamp: 1, + }, + ]); + const messages = cursorRequestMessagesFromRaw(raw); + expect(typeof messages[0]?.content).toBe("string"); + expect(messages[0]?.content).toContain("what color is the sky?"); + expect(messages[0]?.content).toContain(CURSOR_VISION_IMAGE_OMITTED); + expect(await resolveActiveCursorImages(raw)).toEqual([]); + }); + + test("strict base64 rejects truncated and wrong-alphabet payloads", () => { + expect(() => decodeCursorImageDataUrl("data:image/png;base64,iVBOR")).toThrow(CursorImageError); + expect(() => decodeCursorImageDataUrl("data:image/png;base64,!!!!")).toThrow(CursorImageError); + // Signature-only 8-byte stub is valid base64 but must not bypass prepare (no ≤64 passthrough). + const stubUrl = `data:image/png;base64,${Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).toString("base64")}`; + const decoded = decodeCursorImageDataUrl(stubUrl); + expect(decoded.data.byteLength).toBe(8); + }); + + test("signature-only PNG stub is omitted by prepare (no ≤64 bypass)", async () => { + const outcome = await prepareCursorImageForWire({ + data: new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]), + mimeType: "image/png", + uuid: "stub", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("oversize sniffed dimensions omit without Bun decode bomb", async () => { + // PNG IHDR with absurd width/height; sniff rejects before Bun.Image. + const ihdr = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x01, 0x00, 0x00, // width 65536 + 0x00, 0x01, 0x00, 0x00, // height 65536 + ]); + expect(sniffCursorImageDimensions(ihdr)).toEqual({ width: 65536, height: 65536 }); + expect(65536).toBeGreaterThan(MAX_CURSOR_IMAGE_DECODE_EDGE); + expect(65536 * 65536).toBeGreaterThan(MAX_CURSOR_IMAGE_PIXELS); + const outcome = await prepareCursorImageForWire({ + data: ihdr, + mimeType: "image/png", + uuid: "huge", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("createCursorImagePhaseSignal aborts after deadline", async () => { + const phase = createCursorImagePhaseSignal(); + // Use a short local timeout by aborting the parent immediately. + const parent = new AbortController(); + const child = createCursorImagePhaseSignal(parent.signal); + parent.abort(); + expect(child.signal.aborted).toBe(true); + phase.cancel(); + child.cancel(); + }); + + test("truncated FF D8 JPEG under soft cap is omitted (no SOI-only fast path)", async () => { + const truncated = new Uint8Array([0xff, 0xd8, 0x00, 0x00]); + expect(sniffCursorImageFormat(truncated)).toBe("jpeg"); + expect(sniffCursorImageDimensions(truncated)).toBeUndefined(); + const outcome = await prepareCursorImageForWire({ + data: truncated, + mimeType: "image/jpeg", + uuid: "soi-only", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("PNG bytes labeled image/jpeg are re-encoded as JPEG, not passthrough", async () => { + const outcome = await prepareCursorImageForWire({ + data: PNG_BYTES, + mimeType: "image/jpeg", + uuid: "mislabeled", + }); + expect(outcome.status).toBe("ready"); + if (outcome.status !== "ready") throw new Error("expected ready"); + expect(outcome.image.mimeType).toBe("image/jpeg"); + expect(outcome.image.data[0]).toBe(0xff); + expect(outcome.image.data[1]).toBe(0xd8); + expect(sniffCursorImageFormat(outcome.image.data)).toBe("jpeg"); + }); + + test("oversized WebP VP8X header omits before Bun decode", async () => { + // RIFF....WEBP + VP8X with canvas size 65536x65536 (stored as size-1). + const webp = new Uint8Array(30); + webp.set([0x52, 0x49, 0x46, 0x46], 0); // RIFF + webp.set([0x57, 0x45, 0x42, 0x50], 8); // WEBP + webp.set([0x56, 0x50, 0x38, 0x58], 12); // VP8X + // width-1 / height-1 as 24-bit LE at 24..29 → 65535 → displayed 65536 + webp[24] = 0xff; + webp[25] = 0xff; + webp[26] = 0x00; + webp[27] = 0xff; + webp[28] = 0xff; + webp[29] = 0x00; + expect(sniffCursorImageFormat(webp)).toBe("webp"); + expect(sniffCursorImageDimensions(webp)).toEqual({ width: 65536, height: 65536 }); + const outcome = await prepareCursorImageForWire({ + data: webp, + mimeType: "image/webp", + uuid: "huge-webp", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("prepareCursorRawMessages skips historical HTTPS images on a later user turn", async () => { + let httpsFetches = 0; + mockSpy(destinationPolicy, "assessUrlDestination", () => ({ kind: "public" as const })); + mockSpy(destinationPolicy, "resolvePublicAddresses", async () => ({ + addresses: [{ address: "93.184.216.34", family: 4 }], + hostname: "example.com", + })); + mockSpy(imageArtifacts, "pinnedHttpsGet", async () => { + httpsFetches += 1; + return new Response(PNG_BYTES, { + status: 200, + headers: { "content-type": "image/png" }, + }); + }); + + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: "https://example.com/old.png" }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/grok-4.5", + content: [{ type: "text", text: "seen" }], + timestamp: 2, + }, + { role: "user", content: "thanks, no image", timestamp: 3 }, + ]); + expect(httpsFetches).toBe(0); + expect(prepared?.[0]).toEqual({ + role: "user", + content: [{ type: "image", imageUrl: "https://example.com/old.png" }], + timestamp: 1, + }); + expect(cursorVisionPrepareStartIndex(prepared ?? [])).toBe(2); + }); + + test("aborted image-phase signal stops further local prepare work", async () => { + const controller = new AbortController(); + controller.abort(); + await expect(prepareCursorImageForWire({ + data: PNG_BYTES, + mimeType: "image/png", + uuid: "aborted", + }, controller.signal)).rejects.toMatchObject({ name: "AbortError" }); + + await expect(prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "image", imageUrl: PNG_DATA_URL }, + { type: "image", imageUrl: PNG_DATA_URL }, + ], + timestamp: 1, + }, + ], controller.signal)).rejects.toMatchObject({ name: "AbortError" }); + }); +}); + +function actionTextFrom(bytes: Uint8Array): string | undefined { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + return action?.case === "userMessageAction" ? action.value.userMessage?.text : undefined; +} \ No newline at end of file diff --git a/tests/cursor-request-builder.test.ts b/tests/cursor-request-builder.test.ts index 6db299ce1..ff8f94a18 100644 --- a/tests/cursor-request-builder.test.ts +++ b/tests/cursor-request-builder.test.ts @@ -180,7 +180,7 @@ describe("Cursor request builder", () => { ]); }); - test("uses an explicit image placeholder for unsupported image parts", () => { + test("omits image parts from text while preserving other content", () => { const request = createCursorRequest({ ...base, context: { @@ -198,8 +198,78 @@ describe("Cursor request builder", () => { }); expect(request.messages[0]?.content).toContain("see"); - expect(request.messages[0]?.content).toContain("image input unsupported"); - expect(request.messages[0]?.content).toContain("high"); + expect(request.messages[0]?.content).not.toContain("image input unsupported"); + expect(request.messages[0]?.content).not.toContain("data:image/png"); + }); + + test("preserves image-only user turns as empty-string active messages", () => { + const request = createCursorRequest({ + ...base, + context: { + messages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "high" }], + timestamp: 1, + }, + ], + }, + }); + + expect(request.messages).toEqual([{ role: "user", content: "" }]); + expect(request.rawMessages?.length).toBe(1); + }); + + test("preserves image-only active user turn after assistant reply", () => { + const request = createCursorRequest({ + ...base, + context: { + messages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "high" }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,def", detail: "high" }], + timestamp: 3, + }, + ], + }, + }); + + expect(request.messages).toEqual([ + { role: "user", content: "" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "" }, + ]); + expect(request.messages.at(-1)?.role).toBe("user"); + }); + + test("omits assistant messages that carry only tool calls", () => { + const request = createCursorRequest({ + ...base, + context: { + messages: [ + { role: "user", content: "read it", timestamp: 1 }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "toolCall", id: "call_1", name: "read_file", arguments: { path: "a.txt" } }], + timestamp: 2, + }, + ], + }, + }); + + expect(request.messages).toEqual([{ role: "user", content: "read it" }]); }); test("preserves Responses tools and tool choice for Cursor request context", () => { diff --git a/tests/cursor-static-catalog.test.ts b/tests/cursor-static-catalog.test.ts index 4dbd772e8..fa8be304f 100644 --- a/tests/cursor-static-catalog.test.ts +++ b/tests/cursor-static-catalog.test.ts @@ -111,5 +111,12 @@ describe("Cursor static Codex catalog", () => { ]); expect(entries.find(item => item.slug === "cursor/glm-5.2")?.supported_reasoning_levels) .toMatchObject([{ effort: "high" }, { effort: "max" }, { effort: "ultra" }]); + + for (const modelId of ["auto", "composer-2.5", "gpt-5.5", "gemini-3-pro"]) { + expect( + entries.find(item => item.slug === `cursor/${modelId}`)?.input_modalities, + `cursor/${modelId} should advertise image input`, + ).toEqual(["text", "image"]); + } }); }); diff --git a/tests/cursor-vision-wire-harness.test.ts b/tests/cursor-vision-wire-harness.test.ts new file mode 100644 index 000000000..380ab210d --- /dev/null +++ b/tests/cursor-vision-wire-harness.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, test } from "bun:test"; +import { fromBinary } from "@bufbuild/protobuf"; +import { encodeCursorRunRequest } from "../src/adapters/cursor/protobuf-request"; +import { + AgentClientMessageSchema, + ConversationStepSchema, + ConversationTurnStructureSchema, +} from "../src/adapters/cursor/gen/agent_pb"; +import { handleCursorNativeKv, resetCursorBlobStateForTests } from "../src/adapters/cursor/native-exec"; +import { GetBlobArgsSchema, KvServerMessageSchema } from "../src/adapters/cursor/gen/agent_pb"; +import { create } from "@bufbuild/protobuf"; + +function blobData(blobId: Uint8Array): Uint8Array { + const reply = fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id: 1, + message: { case: "getBlobArgs", value: create(GetBlobArgsSchema, { blobId }) }, + }))); + const kv = reply.message.case === "kvClientMessage" ? reply.message.value : undefined; + const result = kv?.message.case === "getBlobResult" ? kv.message.value.blobData : undefined; + if (!result) throw new Error("missing blob data"); + return result; +} + +function activeSelectedImageBytes(bytes: Uint8Array): Uint8Array | undefined { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + const image = action?.case === "userMessageAction" + ? action.value.userMessage?.selectedContext?.selectedImages[0] + : undefined; + if (!image) return undefined; + if (image.dataOrBlobId.case === "data") return image.dataOrBlobId.value; + if (image.dataOrBlobId.case === "blobId") return blobData(image.dataOrBlobId.value); + if (image.dataOrBlobId.case === "blobIdWithData") return image.dataOrBlobId.value.data; + return undefined; +} + +function viewImageToolResultBytes(bytes: Uint8Array): Uint8Array | undefined { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + for (const turnId of run?.conversationState?.turns ?? []) { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") continue; + for (const stepId of turn.turn.value.steps) { + const step = fromBinary(ConversationStepSchema, blobData(stepId)); + if (step.message.case !== "toolCall") continue; + const tool = step.message.value.tool; + if (tool.case !== "mcpToolCall") continue; + const result = tool.value.result?.result; + if (result?.case !== "success") continue; + for (const item of result.value.content) { + if (item.content.case === "image") return item.content.value.data; + } + } + } + return undefined; +} + +describe("Cursor vision wire harness", () => { + test("grok attach + view_image paths keep non-empty PNG bytes on the wire", () => { + resetCursorBlobStateForTests(); + const imageBytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13]); + const imageUrl = `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`; + + const attachBytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "see this" }], + selectedImages: [{ uuid: "img-uuid-1", mimeType: "image/png", data: imageBytes }], + }); + const attachWireBytes = activeSelectedImageBytes(attachBytes); + expect(attachWireBytes).toBeDefined(); + expect(attachWireBytes!.byteLength).toBeGreaterThan(0); + expect(Array.from(attachWireBytes!.slice(0, 4))).toEqual([137, 80, 78, 71]); + + const viewBytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + rawMessages: [ + { role: "user", content: "describe the image", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl, detail: "auto" }], + isError: false, + timestamp: 3, + }, + ], + }); + const viewWireBytes = viewImageToolResultBytes(viewBytes); + expect(viewWireBytes).toBeDefined(); + expect(viewWireBytes!.byteLength).toBeGreaterThan(0); + expect(Array.from(viewWireBytes!)).toEqual(Array.from(imageBytes)); + }); +}); diff --git a/tests/helpers/cursor-grumpy-fixture.png b/tests/helpers/cursor-grumpy-fixture.png new file mode 100644 index 000000000..a06cdc84a Binary files /dev/null and b/tests/helpers/cursor-grumpy-fixture.png differ diff --git a/tests/model-in-list.test.ts b/tests/model-in-list.test.ts new file mode 100644 index 000000000..7dd6399a1 --- /dev/null +++ b/tests/model-in-list.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { modelInList } from "../src/types"; + +describe("modelInList", () => { + test("matches exact ids", () => { + expect(modelInList(["glm-5.2", "auto"], "glm-5.2")).toBe(true); + expect(modelInList(["glm-5.2"], "grok-4.5")).toBe(false); + }); + + test("matches Ollama-style :size family tags", () => { + expect(modelInList(["gpt-oss"], "gpt-oss:120b")).toBe(true); + expect(modelInList(["gpt-oss"], "gpt-oss")).toBe(true); + }); + + test("matches trailing-* prefix entries", () => { + expect(modelInList(["composer-*"], "composer-1")).toBe(true); + expect(modelInList(["composer-*"], "composer-2.5")).toBe(true); + expect(modelInList(["composer-*"], "composer-2.5-fast")).toBe(true); + expect(modelInList(["composer-*"], "composer-9-future")).toBe(true); + expect(modelInList(["composer-*"], "grok-composer-2.5-fast")).toBe(false); + expect(modelInList(["*"], "anything")).toBe(false); + }); +}); diff --git a/tests/oauth-provider-reconcile.test.ts b/tests/oauth-provider-reconcile.test.ts index 611a4e969..62cbf9ce5 100644 --- a/tests/oauth-provider-reconcile.test.ts +++ b/tests/oauth-provider-reconcile.test.ts @@ -5,7 +5,8 @@ import { join } from "node:path"; import { loadConfig } from "../src/config"; import { OAUTH_PROVIDERS, reconcileOAuthProviders, upsertOAuthProvider } from "../src/oauth"; import { getCredential, saveCredential } from "../src/oauth/store"; -import type { OcxConfig } from "../src/types"; +import { CURSOR_NO_VISION_MODELS, cursorModelIds, CURSOR_STATIC_MODELS } from "../src/adapters/cursor/discovery"; +import { modelInList, type OcxConfig } from "../src/types"; const originalHome = process.env.OPENCODEX_HOME; const homes: string[] = []; @@ -170,4 +171,32 @@ describe("OAuth provider reconciliation", () => { upsertOAuthProvider(config, "google-antigravity"); expect(config.providers["google-antigravity"].liveModels).toBe(true); }); + + test("heals a stale Cursor all-models noVisionModels stamp down to the curated list", () => { + const home = mkdtempSync(join(tmpdir(), "ocx-cursor-novision-reconcile-")); + homes.push(home); + process.env.OPENCODEX_HOME = home; + const preset = OAUTH_PROVIDERS.cursor.providerConfig; + const stale = cursorModelIds(CURSOR_STATIC_MODELS); + expect(stale.length).toBeGreaterThan((preset.noVisionModels ?? []).length); + const config = { + port: 10100, + defaultProvider: "cursor", + providers: { + cursor: { + ...structuredClone(preset), + authMode: "oauth", + noVisionModels: [...stale], + }, + }, + } satisfies OcxConfig; + + expect(reconcileOAuthProviders(config)).toBe(true); + expect(config.providers.cursor.noVisionModels).toEqual(preset.noVisionModels); + expect(config.providers.cursor.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(config.providers.cursor.noVisionModels).not.toContain("grok-4.5"); + expect(config.providers.cursor.noVisionModels).toContain("auto"); + expect(modelInList(config.providers.cursor.noVisionModels, "composer-2.5")).toBe(true); + expect(reconcileOAuthProviders(config)).toBe(false); + }); }); diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 5f205e8a3..94457b065 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { buildCatalogEntries } from "../src/codex/catalog"; +import { CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS, cursorModelInputModalities } from "../src/adapters/cursor/discovery"; import { getModelMetadata, resolveMetadataProvider } from "../src/generated/model-metadata"; import { buildInitProviders } from "../src/cli/init"; import { OAUTH_PROVIDERS } from "../src/oauth"; @@ -596,6 +597,18 @@ describe("provider registry parity", () => { expect(seed.modelContextWindows?.["gpt-5.6-luna"]).toBe(1_000_000); expect(seed.modelReasoningEfforts?.["gpt-5.5"]).toEqual(["low", "medium", "high"]); expect(seed.modelReasoningEfforts?.["gpt-5.6-sol"]).toEqual(["low", "medium", "high", "xhigh", "max"]); + expect(cursor?.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(seed.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(seed.modelInputModalities?.auto).toEqual(["text", "image"]); + expect(seed.modelInputModalities?.["composer-2.5"]).toEqual(["text", "image"]); + expect(seed.modelInputModalities?.["gpt-5.5"]).toEqual(["text", "image"]); + expect(seed.modelInputModalities?.["gemini-3-pro"]).toEqual(["text", "image"]); + for (const model of CURSOR_STATIC_MODELS) { + expect( + seed.modelInputModalities?.[model.id], + `Cursor registry seed must advertise image input for ${model.id}`, + ).toEqual(cursorModelInputModalities(CURSOR_STATIC_MODELS)[model.id]); + } const savedCursor: OcxProviderConfig = { adapter: "cursor", baseUrl: "https://api2.cursor.sh" }; enrichProviderFromCatalog("cursor", savedCursor);