From 3393220fabced6c77151480906da0e18087153e7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:54:01 +0000 Subject: [PATCH 01/18] Add verified browser action batches --- packages/agent/src/index.ts | 14 +- packages/agent/src/tools.ts | 36 +- packages/agent/src/translator/browser.ts | 844 ++++++++++++++--- packages/agent/src/translator/translator.ts | 4 +- packages/agent/src/translator/types.ts | 53 +- .../agent/test/translator-browser.test.ts | 872 +++++++++++++++++- packages/ai/src/actions/browser.ts | 156 ++++ packages/ai/src/modes.ts | 10 +- packages/ai/src/providers/common.ts | 5 +- packages/ai/test/modes.test.ts | 60 ++ 10 files changed, 1920 insertions(+), 134 deletions(-) diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 220fff4b..38fa4296 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -6,7 +6,19 @@ export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser"; -export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; +export type { + BatchExecutionResult, + BatchReadResult, + BrowserActObservedSuccessor, + BrowserActOutcome, + BrowserActResult, + BrowserActStepResult, + BrowserActSuccessor, + BrowserActUnavailableSuccessor, + BrowserExpectationEvidence, + BrowserExpectationStatus, + BrowserObservationDiff, +} from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { BatchDetails, diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 9a227434..f35ff9e2 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -15,6 +15,7 @@ import { type TSchema, } from "@onkernel/cua-ai"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; +import type { BrowserActResult } from "./translator/types"; import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core"; export interface ComputerToolOptions { @@ -37,6 +38,7 @@ export interface BatchDetails { | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number } | { type: "browser_text"; label: string; bytes: number } + | { type: "browser_act"; result: BrowserActResult } >; } @@ -170,6 +172,9 @@ async function executeBatchTool( } else if (read.type === "browser_text") { readResults.push({ type: "browser_text", label: read.label, bytes: read.text.length }); content.push({ type: "text", text: read.text }); + } else if (read.type === "browser_act") { + readResults.push({ type: "browser_act", result: read.result }); + content.push({ type: "text", text: formatBrowserActResult(read.result) }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType }); @@ -185,7 +190,16 @@ async function executeBatchTool( } catch (err) { throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err }); } - return { content, details: { statusText: "Actions executed successfully.", readResults } }; + const actResults = readResults.flatMap((read) => (read.type === "browser_act" ? [read.result] : [])); + const statusText = + actResults.length === 0 + ? "Actions executed successfully." + : actResults.some((result) => result.outcome === "didnt") + ? "Browser actions did not satisfy their expectations." + : actResults.every((result) => result.outcome === "worked") + ? "Browser actions worked." + : "Browser action outcome is unknown."; + return { content, details: { statusText, readResults } }; } async function executeNavigationTool( @@ -257,6 +271,26 @@ async function executePlaywrightTool(translator: InternalComputerTranslator, par } } +function formatBrowserActResult(result: BrowserActResult): string { + const lines = [`browser_act outcome: ${result.outcome}`]; + if (result.stopped_at !== undefined) lines.push(`stopped_at: ${result.stopped_at} (${result.stop_reason ?? "unknown"})`); + for (const step of result.steps) { + lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.evidence.join("; ")}`); + for (const detail of step.expectation?.details ?? []) lines.push(` ${detail}`); + } + if (result.final_expectation) lines.push(`final expectation: ${result.final_expectation.status}`); + if (result.successor.status === "unavailable") { + lines.push(`successor unavailable: ${result.successor.error}`); + } else { + const { diff } = result.successor; + lines.push(`successor diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`); + for (const line of diff.added) lines.push(` + ${line}`); + for (const line of diff.removed) lines.push(` - ${line}`); + lines.push(result.successor.text); + } + return lines.join("\n"); +} + function formatPlaywrightResult(result: unknown): string { return typeof result === "string" ? result : JSON.stringify(result); } diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 9cb21a14..7ce3e857 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -1,5 +1,8 @@ import { normalizeGotoUrl, + type CuaActionBrowserAct, + type CuaBrowserActStep, + type CuaBrowserExpectation, type CuaActionBrowserClick, type CuaActionBrowserDrag, type CuaActionBrowserFill, @@ -13,13 +16,21 @@ import { type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; -import type { BatchReadResult } from "./types"; +import type { + BatchReadResult, + BrowserActResult, + BrowserActStepResult, + BrowserExpectationEvidence, + BrowserObservationDiff, +} from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; const SCROLL_NOTCH_PX = 120; +const EXPECTATION_TIMEOUT_MS = 2_000; +const EXPECTATION_POLL_MS = 50; const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; const REF_PLACEHOLDER = "\u0000"; @@ -42,6 +53,8 @@ interface RefEntry { targetId: string; /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ frameId: string; + /** Target whose CDP session owns this frame; omitted only by ref state exported before this field existed. */ + sessionTargetId?: string; /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ sessionId: string; generation: number; @@ -60,25 +73,53 @@ interface NthIndex { interface RenderContext { targetId: string; frameKey: string; + sessionTargetId: string; sessionId: string; generation: number; - interactiveOnly: boolean; nthIndex: NthIndex; cursorIds?: ReadonlySet; } -interface RenderedLine { +interface ObservationLine { text: string; refNode?: AXNode; ctx: RenderContext; } +interface ObservedNode { + node: AXNode; + ctx: RenderContext; +} + +interface BrowserObservation { + targetId: string; + tree: FrameStitch; + stitches: Map; + nodes: ObservedNode[]; + url: string; + title: string; + generations: Map; + complete: boolean; +} + +interface BrowserPresentation { + observation: BrowserObservation; + key: string; + lines: ObservationLine[]; + shape: string; +} + interface FrameStitch { byId: Map; roots: string[]; ctx: RenderContext; } +interface FrameStitchResult { + frames: Map; + complete: boolean; +} + export interface BrowserFindCandidate { ref: string; role: string; @@ -99,6 +140,7 @@ export interface BrowserRefState { activeTargetId?: string; generations: Array<[string, number]>; refs: Array<[string, Omit]>; + frameParents?: Array<[string, string]>; } /** @@ -135,7 +177,8 @@ export class BrowserExecutor { private readonly targetsBySession = new Map(); private readonly frameSessions = new Map(); private readonly frameTargets = new Set(); - private readonly lastSnapshots = new Map(); + private readonly frameParents = new Map(); + private readonly lastSnapshots = new Map(); private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; private refCounter = 0; @@ -155,10 +198,7 @@ export class BrowserExecutor { const targetId = this.targetsBySession.get(event.sessionId); if (!targetId) return; if (this.frameTargets.has(targetId)) { - // Refs from a frame target's tree (its root and any same-process - // subframes inlined in it) are all minted against the target's key, - // so any navigation observed in its session stales them. - this.invalidateFrame(targetId); + this.invalidateSessionTarget(targetId); return; } if (frame.parentId) { @@ -223,6 +263,7 @@ export class BrowserExecutor { ...(this.activeTargetId ? { activeTargetId: this.activeTargetId } : {}), generations: [...this.generations], refs: [...this.refs].map(([ref, { sessionId: _sessionId, ...entry }]) => [ref, entry]), + frameParents: [...this.frameParents], }; } @@ -232,6 +273,7 @@ export class BrowserExecutor { this.activeTargetId = state.activeTargetId ?? this.activeTargetId; for (const [frameId, generation] of state.generations) this.generations.set(frameId, generation); for (const [ref, entry] of state.refs) this.refs.set(ref, { ...entry, sessionId: "" }); + for (const [frameId, parentId] of state.frameParents ?? []) this.frameParents.set(frameId, parentId); } async execute(action: CuaBrowserAction): Promise { @@ -245,6 +287,8 @@ export class BrowserExecutor { switch (action.type) { case "browser_snapshot": return [{ type: "browser_text", label: "snapshot", text: await this.snapshot(action) }]; + case "browser_act": + return [{ type: "browser_act", result: await this.act(action) }]; case "browser_text": return [{ type: "browser_text", label: "text", text: await this.pageText(tabOf(action)) }]; case "browser_find": @@ -306,45 +350,116 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - const targetId = await this.resolveTarget(action.tab_id); - const pageSession = await this.attach(targetId); - const refEntry = action.ref ? this.resolveRef(action.ref, targetId) : undefined; - const frameKey = refEntry?.frameId ?? targetId; - const { nodes, sessionId } = await this.frameAxTree(frameKey, targetId, pageSession); - const byId = new Map(nodes.map((node) => [node.nodeId, node])); - let rootIds = nodes.filter((node) => !node.parentId).map((node) => node.nodeId); - if (action.ref && refEntry) { - const rootNode = - nodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, nodes); - rootIds = [rootNode.nodeId]; + const presentation = this.presentObservation(await this.observe(action.tab_id), action); + const cached = this.lastSnapshots.get(presentation.observation.targetId); + this.lastSnapshots.set(presentation.observation.targetId, presentation); + if (cached && cached.key === presentation.key && cached.shape === presentation.shape) return UNCHANGED_SNAPSHOT; + return this.renderObservation(presentation); + } + + /** Build a generation-fenced observation before applying presentation options or minting refs. */ + private async observe(tabId?: string): Promise { + let changed: ObservationChangedError | undefined; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + return await this.collectObservation(tabId); + } catch (err) { + if (!(err instanceof ObservationChangedError)) throw err; + changed = err; + } } + throw changed ?? new ObservationChangedError(); + } - const interactiveOnly = action.filter === "interactive"; + private async collectObservation(tabId?: string): Promise { + const targetId = await this.resolveTarget(tabId); + const pageSession = await this.attach(targetId); + const targetsBefore = await this.cdp.pageTargets(); + const targetBefore = targetsBefore.find((candidate) => candidate.targetId === targetId); + if (!targetBefore) throw new ObservationChangedError(); + const generationBefore = this.generation(targetId); + const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); const ctx: RenderContext = { targetId, - frameKey, + frameKey: targetId, + sessionTargetId: targetId, sessionId, - generation: this.generation(frameKey), - interactiveOnly, + generation: generationBefore, nthIndex: buildNthIndex(nodes), - cursorIds: frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, + cursorIds: await this.cursorPointerIds(pageSession), + }; + const tree: FrameStitch = { + byId: new Map(nodes.map((node) => [node.nodeId, node])), + roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), + ctx, + }; + const { frames: stitches, complete } = await this.stitchFrames(nodes, targetId, pageSession); + const observedNodes: ObservedNode[] = nodes.map((node) => ({ node, ctx })); + for (const stitch of stitches.values()) { + for (const node of stitch.byId.values()) observedNodes.push({ node, ctx: stitch.ctx }); + } + const targetsAfter = await this.cdp.pageTargets(); + const targetAfter = targetsAfter.find((candidate) => candidate.targetId === targetId); + if ( + !targetAfter || + generationBefore !== this.generation(targetId) || + targetBefore.url !== targetAfter.url || + targetBefore.title !== targetAfter.title + ) { + throw new ObservationChangedError(); + } + const generations = new Map([[targetId, ctx.generation]]); + for (const stitch of stitches.values()) generations.set(stitch.ctx.frameKey, stitch.ctx.generation); + if ([...generations].some(([frameKey, generation]) => this.generation(frameKey) !== generation)) { + throw new ObservationChangedError(); + } + return { + targetId, + tree, + stitches, + nodes: observedNodes, + url: targetAfter.url, + title: targetAfter.title, + generations, + complete, }; - const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); - const lines: RenderedLine[] = []; + } + + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { + let tree = observation.tree; + let rootIds = tree.roots; + if (action.ref) { + const entry = this.resolveRef(action.ref, observation.targetId); + const frameTree = + entry.frameId === observation.tree.ctx.frameKey + ? observation.tree + : [...observation.stitches.values()].find((candidate) => candidate.ctx.frameKey === entry.frameId); + if (!frameTree) throw staleRefError(action.ref); + tree = frameTree; + const nodes = [...tree.byId.values()]; + const rootNode = nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId) ?? this.healEntry(action.ref, entry, nodes); + rootIds = [rootNode.nodeId]; + } + + const interactiveOnly = action.filter === "interactive"; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const walk = (tree: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { - const node = tree.get(nodeId); + const lines: ObservationLine[] = []; + const walk = (nodes: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { + const node = nodes.get(nodeId); if (!node) return; let childDepth = depth; if (!node.ignored) { - const rendered = this.renderNode(node, depth, parentName, treeCtx); + const rendered = this.renderNode(node, depth, parentName, treeCtx, interactiveOnly); if (rendered) { lines.push({ ...rendered, ctx: treeCtx }); childDepth = depth + 1; } } if (childDepth > maxDepth) return; - const stitch = treeCtx === ctx && node.backendDOMNodeId !== undefined ? stitches.get(node.backendDOMNodeId) : undefined; + const stitch = + node.backendDOMNodeId === undefined + ? undefined + : observation.stitches.get(frameStitchKey(treeCtx.frameKey, node.backendDOMNodeId)); if (stitch) { for (const frameRootId of stitch.roots) walk(stitch.byId, stitch.ctx, frameRootId, childDepth, ""); return; @@ -352,28 +467,31 @@ export class BrowserExecutor { const name = node.name?.value ?? ""; const childName = name || parentName; const childIds = node.childIds ?? []; - for (let i = 0; i < childIds.length; i += 1) { - const run = staticTextRun(tree, childIds, i); + for (let index = 0; index < childIds.length; index += 1) { + const run = staticTextRun(nodes, childIds, index); if (run) { - const rendered = this.renderNode(run.node, childDepth, childName, treeCtx); + const rendered = this.renderNode(run.node, childDepth, childName, treeCtx, interactiveOnly); if (rendered) lines.push({ ...rendered, ctx: treeCtx }); - i = run.end; + index = run.end; continue; } - walk(tree, treeCtx, childIds[i]!, childDepth, childName); + walk(nodes, treeCtx, childIds[index]!, childDepth, childName); } }; - for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); + for (const rootId of rootIds) walk(tree.byId, tree.ctx, rootId, 0, ""); - const shape = lines.map((line) => line.text).join("\n"); - const frameGenerations = [...stitches.values()].map((stitch) => `${stitch.ctx.frameKey}:${stitch.ctx.generation}`); - const key = [action.ref ?? "", action.depth ?? "", action.filter ?? "", `${frameKey}:${ctx.generation}`, ...frameGenerations].join("|"); - const cached = this.lastSnapshots.get(targetId); - this.lastSnapshots.set(targetId, { key, shape }); - if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; + const frameGenerations = [...observation.generations].map(([key, generation]) => `${key}:${generation}`); + return { + observation, + key: [action.ref ?? "", action.depth ?? "", action.filter ?? "", ...frameGenerations].join("|"), + lines, + shape: lines.map((line) => line.text).join("\n"), + }; + } + private renderObservation(presentation: BrowserPresentation): string { let text = ""; - for (const line of lines) { + for (const line of presentation.lines) { if (text.length > SNAPSHOT_CHAR_LIMIT) break; const rendered = line.refNode ? line.text.replace(REF_PLACEHOLDER, this.mintRef(line.refNode, line.ctx)) : line.text; text = text ? `${text}\n${rendered}` : rendered; @@ -381,18 +499,371 @@ export class BrowserExecutor { if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } - this.pruneRefs(targetId); + this.pruneRefs(presentation.observation.targetId); return text || "(empty accessibility tree)"; } - private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): { text: string; refNode?: AXNode } | undefined { + private async act(action: CuaActionBrowserAct): Promise { + const observationAction: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor }; + const completeObservationAction: CuaActionBrowserSnapshot = { + type: "browser_snapshot", + tab_id: action.tab_id, + depth: Number.MAX_SAFE_INTEGER, + }; + const baseline = await this.observe(action.tab_id); + let current = baseline; + let currentTargets = await this.pageTargetIds(); + let currentDialogCount = this.dialogNotes.length; + const steps: BrowserActStepResult[] = []; + let stoppedAt: number | undefined; + let stopReason: BrowserActResult["stop_reason"]; + + for (let index = 0; index < action.steps.length; index += 1) { + const step = action.steps[index]!; + const evidence: string[] = []; + let beforeObservation: BrowserObservation; + let beforeTargets: string[]; + try { + beforeObservation = await this.observe(action.tab_id); + beforeTargets = await this.pageTargetIds(); + } catch (err) { + evidence.push(`pre-action observation failed: ${errorMessage(err)}`); + steps.push({ index, type: step.type, outcome: "unknown", evidence }); + stoppedAt = index; + stopReason = "control_flow"; + break; + } + + const boundary = browserControlChange( + current, + beforeObservation, + this.generations, + currentTargets, + beforeTargets, + currentDialogCount, + this.dialogNotes.length, + ); + current = beforeObservation; + currentTargets = beforeTargets; + currentDialogCount = this.dialogNotes.length; + if (boundary) { + evidence.push(`${boundary} detected before input delivery`); + steps.push({ index, type: step.type, outcome: "unknown", evidence }); + stoppedAt = index; + stopReason = boundary; + break; + } + + const before = step.expect ? this.evaluateExpectation(step.expect, beforeObservation, baseline) : undefined; + const dialogCount = currentDialogCount; + let actionError: unknown; + try { + await this.executeActStep(step, action.tab_id); + evidence.push("input delivered"); + } catch (err) { + actionError = err; + evidence.push(errorMessage(err)); + } + + let expectation: BrowserExpectationEvidence | undefined; + let observationError: unknown; + let boundaryAfter: BrowserActResult["stop_reason"]; + const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; + while (true) { + try { + await delay(0); + const afterObservation = await this.observe(action.tab_id); + await delay(0); + const afterTargets = await this.pageTargetIds(); + boundaryAfter = browserControlChange( + beforeObservation, + afterObservation, + this.generations, + beforeTargets, + afterTargets, + dialogCount, + this.dialogNotes.length, + ); + current = afterObservation; + currentTargets = afterTargets; + currentDialogCount = this.dialogNotes.length; + if (step.expect) expectation = expectationEvidence(before!, this.evaluateExpectation(step.expect, current, baseline)); + if (boundaryAfter && expectation?.status === "failed") { + expectation = { ...expectation, status: "unverifiable", details: [...expectation.details, `${boundaryAfter} interrupted verification`] }; + } + if (boundaryAfter || !step.expect || expectation?.status !== "failed" || Date.now() >= deadline) break; + await delay(EXPECTATION_POLL_MS); + } catch (err) { + observationError = err; + if (step.expect) { + expectation = { + status: "unverifiable", + before: before?.truth, + details: [...(before?.details ?? []), errorMessage(err)], + }; + } + break; + } + } + + let outcome: BrowserActStepResult["outcome"] = "unknown"; + const staleRef = actionError !== undefined && /ref .* stale/.test(errorMessage(actionError)); + if (expectation?.status === "newly_verified" && actionError === undefined) outcome = "worked"; + else if (expectation?.status === "failed" || staleRef) outcome = "didnt"; + if (expectation) evidence.push(`expectation ${expectation.status}`); + if (observationError) evidence.push(`post-action observation failed: ${errorMessage(observationError)}`); + steps.push({ index, type: step.type, outcome, evidence, ...(expectation ? { expectation } : {}) }); + + if (boundaryAfter) { + stopReason = boundaryAfter; + } else if (observationError || expectation?.status === "unverifiable") { + stopReason = "control_flow"; + } else if (actionError) { + stopReason = staleRef ? "stale_ref" : "action_failed"; + } else if (expectation?.status === "failed") { + stopReason = "expectation_failed"; + } + if (stopReason) { + stoppedAt = index; + break; + } + } + + let finalExpectation: BrowserExpectationEvidence | undefined; + const terminalNavigation = + stopReason === "navigation" && + stoppedAt === action.steps.length - 1 && + steps.length === action.steps.length && + steps.at(-1)?.evidence.includes("input delivered") === true; + if (action.expect && (!stopReason || terminalNavigation)) { + const before = this.evaluateExpectation(action.expect, baseline, baseline); + const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; + while (true) { + try { + const afterObservation = await this.observe(action.tab_id); + await delay(0); + const afterTargets = await this.pageTargetIds(); + const boundary = browserControlChange( + current, + afterObservation, + this.generations, + currentTargets, + afterTargets, + currentDialogCount, + this.dialogNotes.length, + ); + current = afterObservation; + currentTargets = afterTargets; + currentDialogCount = this.dialogNotes.length; + if (boundary) { + finalExpectation = { + status: "unverifiable", + before: before.truth, + details: [...(finalExpectation?.details ?? before.details), `${boundary} interrupted verification`], + }; + stopReason = boundary; + break; + } + finalExpectation = expectationEvidence(before, this.evaluateExpectation(action.expect, current, baseline)); + if (finalExpectation.status !== "failed" || Date.now() >= deadline) break; + await delay(EXPECTATION_POLL_MS); + } catch (err) { + finalExpectation = { status: "unverifiable", before: before.truth, details: [...before.details, errorMessage(err)] }; + stopReason = "control_flow"; + break; + } + } + if (!stopReason && finalExpectation?.status === "failed") stopReason = "expectation_failed"; + else if (!stopReason && finalExpectation?.status === "unverifiable") stopReason = "control_flow"; + if (stopReason) stoppedAt = action.steps.length; + } + + let successor: BrowserActResult["successor"] | undefined; + let successorError: unknown; + for (let attempt = 0; attempt < 3 && !successor; attempt += 1) { + try { + const successorObservation = await this.observe(action.tab_id); + await delay(0); + const successorTargets = await this.pageTargetIds(); + const lateBoundary = browserControlChange( + current, + successorObservation, + this.generations, + currentTargets, + successorTargets, + currentDialogCount, + this.dialogNotes.length, + ); + current = successorObservation; + currentTargets = successorTargets; + currentDialogCount = this.dialogNotes.length; + if (lateBoundary) { + if (!stopReason || stopReason === "control_flow") stopReason = lateBoundary; + stoppedAt ??= action.steps.length; + successorError = new ObservationChangedError(); + continue; + } + const baselineComplete = this.presentObservation(baseline, completeObservationAction); + const currentComplete = this.presentObservation(current, completeObservationAction); + const currentPresentation = this.presentObservation(current, observationAction); + successor = { + status: "observed", + text: this.renderObservation(currentPresentation), + url: current.url, + title: current.title, + diff: diffObservations(baselineComplete, currentComplete), + }; + this.lastSnapshots.set(current.targetId, currentPresentation); + } catch (err) { + successorError = err; + } + } + if (!successor) { + successor = { status: "unavailable", error: errorMessage(successorError ?? new ObservationChangedError()) }; + stopReason ??= "control_flow"; + stoppedAt ??= action.steps.length; + } + + const completed = steps.length === action.steps.length && (!stopReason || terminalNavigation); + const definitiveFailure = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed"; + const semanticallyVerified = action.expect + ? finalExpectation?.status === "newly_verified" + : steps.length > 0 && steps.every((step) => step.outcome === "worked"); + const outcome = definitiveFailure ? "didnt" : completed && semanticallyVerified ? "worked" : "unknown"; + return { + outcome, + steps, + ...(stoppedAt !== undefined ? { stopped_at: stoppedAt } : {}), + ...(stopReason ? { stop_reason: stopReason } : {}), + ...(finalExpectation ? { final_expectation: finalExpectation } : {}), + successor, + }; + } + + private async executeActStep(step: CuaBrowserActStep, tabId?: string): Promise { + switch (step.type) { + case "click": + return this.click({ + type: "browser_click", + tab_id: tabId, + ref: step.ref, + button: step.button, + num_clicks: step.num_clicks, + modifiers: step.modifiers, + }); + case "hover": + return this.hover({ type: "browser_hover", tab_id: tabId, ref: step.ref }); + case "fill": + return this.fill({ type: "browser_fill", tab_id: tabId, ref: step.ref, value: step.value }); + case "type": { + const session = await this.session(tabId); + await this.cdp.send("Input.insertText", { text: step.text }, session); + return; + } + case "key": + return this.key({ type: "browser_key", tab_id: tabId, text: step.text, repeat: step.repeat }); + case "scroll_to": + return this.scrollTo({ type: "browser_scroll_to", tab_id: tabId, ref: step.ref }); + case "wait": + await new Promise((resolve) => setTimeout(resolve, Math.min(Math.max(0, step.ms ?? 1000), 30_000))); + } + } + + private evaluateExpectation( + expectation: CuaBrowserExpectation, + observation: BrowserObservation, + baseline: BrowserObservation, + ): ExpectationEvaluation { + if ("all" in expectation) { + const children = expectation.all.map((child) => this.evaluateExpectation(child, observation, baseline)); + const truth = children.some((child) => child.truth === false) + ? false + : children.some((child) => child.truth === undefined) + ? undefined + : true; + return { truth, details: children.flatMap((child) => child.details) }; + } + if ("any" in expectation) { + const children = expectation.any.map((child) => this.evaluateExpectation(child, observation, baseline)); + const truth = children.some((child) => child.truth === true) + ? true + : children.some((child) => child.truth === undefined) + ? undefined + : false; + return { truth, details: children.flatMap((child) => child.details) }; + } + if (expectation.type === "text") { + const found = observation.nodes.some( + ({ node }) => !node.ignored && (node.name?.value ?? "").toLowerCase().includes(expectation.text.toLowerCase()), + ); + const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); + const completeness = observation.complete ? "" : "; observation incomplete"; + return { truth, details: [`text ${JSON.stringify(expectation.text)} ${found ? "present" : "absent"}${completeness}`] }; + } + if (expectation.type === "role_name") { + const found = observation.nodes.some( + ({ node }) => + !node.ignored && + (expectation.role === undefined || (node.role?.value ?? "") === expectation.role) && + (expectation.name === undefined || (node.name?.value ?? "") === expectation.name), + ); + const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); + const completeness = observation.complete ? "" : "; observation incomplete"; + return { truth, details: [`role/name ${found ? "present" : "absent"}${completeness}`] }; + } + if (expectation.type === "url" || expectation.type === "title") { + const value = observation[expectation.type]; + const initial = baseline[expectation.type]; + const checks = [ + expectation.equals === undefined || value === expectation.equals, + expectation.contains === undefined || value.includes(expectation.contains), + expectation.changed === undefined || (value !== initial) === expectation.changed, + ]; + return { truth: checks.every(Boolean), details: [`${expectation.type}=${JSON.stringify(value)}`] }; + } + if (expectation.type !== "ref") return { truth: undefined, details: ["unsupported expectation"] }; + + const entry = this.refs.get(expectation.ref); + if (!entry || entry.targetId !== observation.targetId || entry.generation !== this.generation(entry.frameId)) { + return { truth: undefined, details: [`ref ${expectation.ref} is stale`] }; + } + const frameNodes = observation.nodes.filter(({ ctx }) => ctx.frameKey === entry.frameId).map(({ node }) => node); + let observed = frameNodes.find((node) => node.backendDOMNodeId === entry.backendNodeId); + if (!observed) { + try { + observed = this.healEntry(expectation.ref, entry, frameNodes); + } catch { + return { truth: undefined, details: [`ref ${expectation.ref} was not observable`] }; + } + } + const checks: boolean[] = []; + if (expectation.value !== undefined) checks.push(String(observed.value?.value ?? "") === expectation.value); + for (const state of ["checked", "selected", "expanded"] as const) { + const expected = expectation[state]; + if (expected === undefined) continue; + const actual = observed.properties?.find((property) => property.name === state)?.value?.value; + checks.push(normalizeState(actual) === expected); + } + return { + truth: checks.length > 0 ? checks.every(Boolean) : undefined, + details: [`ref ${expectation.ref} value/state ${checks.every(Boolean) ? "matched" : "did not match"}`], + }; + } + + private renderNode( + node: AXNode, + depth: number, + parentName: string, + ctx: RenderContext, + interactiveOnly: boolean, + ): { text: string; refNode?: AXNode } | undefined { const role = node.role?.value ?? ""; const name = node.name?.value ?? ""; const interactive = INTERACTIVE_ROLES.has(role); const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); - if (ctx.interactiveOnly && !interactive && !pointer) return undefined; + if (interactiveOnly && !interactive && !pointer) return undefined; if (role === "StaticText" && name === parentName) return undefined; - if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; + if (!interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; let refNode: AXNode | undefined; const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); @@ -418,42 +889,58 @@ export class BrowserExecutor { return { nodes, sessionId: pageSession }; } - /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ - private async stitchFrames( - nodes: AXNode[], - targetId: string, - pageSession: string, - interactiveOnly: boolean, - ): Promise> { - const stitches = new Map(); - for (const node of nodes) { - if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; - try { - const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( - "DOM.describeNode", - { backendNodeId: node.backendDOMNodeId, depth: 1 }, - pageSession, - ); - const frameId = dom.contentDocument?.frameId ?? dom.frameId; - if (!frameId || frameId === targetId) continue; - const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); - stitches.set(node.backendDOMNodeId, { - byId: new Map(frameNodes.map((frameNode) => [frameNode.nodeId, frameNode])), - roots: frameNodes.filter((frameNode) => !frameNode.parentId).map((frameNode) => frameNode.nodeId), - ctx: { - targetId, - frameKey: frameId, - sessionId, - generation: this.generation(frameId), - interactiveOnly, - nthIndex: buildNthIndex(frameNodes), - }, - }); - } catch { - // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. + /** Resolve iframe nodes recursively and fetch each child frame's AX tree for stitching. */ + private async stitchFrames(nodes: AXNode[], targetId: string, pageSession: string): Promise { + const stitches = new Map(); + const visitedFrames = new Set([targetId]); + let complete = true; + const visit = async ( + frameNodes: AXNode[], + parentFrameKey: string, + parentSession: string, + parentSessionTargetId: string, + ): Promise => { + for (const node of frameNodes) { + if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; + try { + const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( + "DOM.describeNode", + { backendNodeId: node.backendDOMNodeId, depth: 1 }, + parentSession, + ); + const frameId = dom.contentDocument?.frameId ?? dom.frameId; + if (!frameId || visitedFrames.has(frameId)) { + complete = false; + continue; + } + visitedFrames.add(frameId); + this.frameParents.set(frameId, parentFrameKey); + const generation = this.generation(frameId); + const { nodes: childNodes, sessionId } = await this.frameAxTree(frameId, targetId, parentSession); + if (generation !== this.generation(frameId)) throw new ObservationChangedError(); + const sessionTargetId = this.frameSessions.has(frameId) ? frameId : parentSessionTargetId; + const stitch: FrameStitch = { + byId: new Map(childNodes.map((child) => [child.nodeId, child])), + roots: childNodes.filter((child) => !child.parentId).map((child) => child.nodeId), + ctx: { + targetId, + frameKey: frameId, + sessionTargetId, + sessionId, + generation, + nthIndex: buildNthIndex(childNodes), + }, + }; + stitches.set(frameStitchKey(parentFrameKey, node.backendDOMNodeId), stitch); + await visit(childNodes, frameId, sessionId, sessionTargetId); + } catch (err) { + if (err instanceof ObservationChangedError) throw err; + complete = false; + } } - } - return stitches; + }; + await visit(nodes, targetId, pageSession, targetId); + return { frames: stitches, complete }; } /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ @@ -499,35 +986,13 @@ export class BrowserExecutor { * matches, best first. Structured counterpart of the `browser_find` action. */ async findCandidates(query: string, tabId?: string, roles?: ReadonlySet): Promise { - const targetId = await this.resolveTarget(tabId); - const session = await this.attach(targetId); - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); - const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ - { - nodes, - ctx: { - targetId, - frameKey: targetId, - sessionId: session, - generation: this.generation(targetId), - interactiveOnly: false, - nthIndex: buildNthIndex(nodes), - }, - }, - ]; - // Search stitched frames too so find sees everything snapshot renders. - for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { - pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); - } + const observation = await this.observe(tabId); const queryTokens = tokenize(query); - const scored = pools - .flatMap(({ nodes: poolNodes, ctx }) => - poolNodes - .filter( - (node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), - ) - .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), + const scored = observation.nodes + .filter( + ({ node }) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), ) + .map(({ node, ctx }) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })) .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); @@ -537,7 +1002,7 @@ export class BrowserExecutor { name: node.name?.value ?? "", score, })); - this.pruneRefs(targetId); + this.pruneRefs(observation.targetId); return candidates; } @@ -810,6 +1275,7 @@ export class BrowserExecutor { backendNodeId: node.backendDOMNodeId!, targetId: ctx.targetId, frameId: ctx.frameKey, + sessionTargetId: ctx.sessionTargetId, sessionId: ctx.sessionId, generation: ctx.generation, role, @@ -828,7 +1294,13 @@ export class BrowserExecutor { */ private async refSession(entry: RefEntry): Promise { if (!entry.sessionId) { - entry.sessionId = this.frameSessions.get(entry.frameId) ?? (await this.attach(entry.targetId)); + await this.attach(entry.targetId); + const sessionTargetId = entry.sessionTargetId ?? entry.frameId; + const frameSession = this.frameSessions.get(sessionTargetId); + if (sessionTargetId !== entry.targetId && !frameSession) { + throw new Error("owning frame session is unavailable; the element ref is stale"); + } + entry.sessionId = frameSession ?? (await this.attach(entry.targetId)); } return entry.sessionId; } @@ -855,30 +1327,50 @@ export class BrowserExecutor { } private invalidateFrame(frameKey: string): void { - let tracked = this.generations.has(frameKey) || this.frameSessions.has(frameKey); - for (const [ref, entry] of this.refs) { - if (entry.frameId === frameKey) { - this.refs.delete(ref); - tracked = true; + const invalidated = new Set([frameKey]); + let found = true; + while (found) { + found = false; + for (const [child, parent] of this.frameParents) { + if (invalidated.has(parent) && !invalidated.has(child)) { + invalidated.add(child); + found = true; + } } } - // Frames we never referenced don't get a generation entry, or pages with - // rotating ad iframes would grow the map without bound. - if (tracked) this.generations.set(frameKey, this.generation(frameKey) + 1); + for (const invalidatedFrame of invalidated) { + this.generations.set(invalidatedFrame, this.generation(invalidatedFrame) + 1); + if (invalidatedFrame !== frameKey) this.frameParents.delete(invalidatedFrame); + } + for (const [ref, entry] of this.refs) { + if (invalidated.has(entry.frameId)) this.refs.delete(ref); + } + } + + private invalidateSessionTarget(targetId: string): void { + const ownedFrames = new Set(); + for (const entry of this.refs.values()) { + if (entry.sessionTargetId === targetId) ownedFrames.add(entry.frameId); + } + this.invalidateFrame(targetId); + for (const frameKey of ownedFrames) this.invalidateFrame(frameKey); } private dropTarget(targetId: string): void { - this.generations.delete(targetId); + this.invalidateFrame(targetId); this.selfNavigations.delete(targetId); this.lastSnapshots.delete(targetId); this.frameSessions.delete(targetId); this.frameTargets.delete(targetId); + this.frameParents.delete(targetId); + const invalidatedFrames = new Set(); for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId || entry.frameId === targetId) { - if (entry.frameId !== targetId) this.generations.delete(entry.frameId); + if (entry.targetId === targetId || entry.frameId === targetId || entry.sessionTargetId === targetId) { + if (entry.frameId !== targetId) invalidatedFrames.add(entry.frameId); this.refs.delete(ref); } } + for (const frameId of invalidatedFrames) this.generations.set(frameId, this.generation(frameId) + 1); } /** SPAs can mint refs indefinitely without ever navigating; bound per-target growth by evicting the oldest. */ @@ -911,6 +1403,10 @@ export class BrowserExecutor { return session; } + private async pageTargetIds(): Promise { + return (await this.cdp.pageTargets()).map((target) => target.targetId).sort(); + } + private async resolveTarget(tabId?: string): Promise { const targets = await this.cdp.pageTargets(); if (targets.length === 0) throw new Error("no open browser tabs"); @@ -928,6 +1424,106 @@ export class BrowserExecutor { } } +class ObservationChangedError extends Error { + constructor() { + super("page changed while collecting the browser observation"); + } +} + +interface ExpectationEvaluation { + truth?: boolean; + details: string[]; +} + +function expectationEvidence(before: ExpectationEvaluation, after: ExpectationEvaluation): BrowserExpectationEvidence { + const details = [...before.details.map((detail) => `before: ${detail}`), ...after.details.map((detail) => `after: ${detail}`)]; + if (before.truth === undefined || after.truth === undefined) { + return { status: "unverifiable", before: before.truth, after: after.truth, details }; + } + if (before.truth && after.truth) return { status: "preexisting", before: true, after: true, details }; + if (!before.truth && after.truth) return { status: "newly_verified", before: false, after: true, details }; + return { status: "failed", before: before.truth, after: after.truth, details }; +} + +function normalizeState(value: unknown): boolean | "mixed" | undefined { + if (value === true || value === "true") return true; + if (value === false || value === "false") return false; + if (value === "mixed") return "mixed"; + return undefined; +} + +function observationGenerationsChanged( + before: BrowserObservation, + after: BrowserObservation, + live: ReadonlyMap, +): boolean { + const keys = new Set([...before.generations.keys(), ...after.generations.keys()]); + return [...keys].some( + (key) => + before.generations.get(key) !== after.generations.get(key) || + (after.generations.has(key) && after.generations.get(key) !== (live.get(key) ?? 0)), + ); +} + +function browserControlChange( + before: BrowserObservation, + after: BrowserObservation, + liveGenerations: ReadonlyMap, + beforeTargets: readonly string[], + afterTargets: readonly string[], + beforeDialogCount: number, + afterDialogCount: number, +): BrowserActResult["stop_reason"] { + if (afterDialogCount > beforeDialogCount) return "dialog"; + if (observationGenerationsChanged(before, after, liveGenerations)) return "navigation"; + if (targetsChanged(beforeTargets, afterTargets)) return "control_flow"; + return undefined; +} + +function targetsChanged(before: readonly string[], after: readonly string[]): boolean { + return before.length !== after.length || before.some((targetId, index) => targetId !== after[index]); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff { + const oldLines = before.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); + const newLines = after.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); + const remaining = new Map(); + for (const line of oldLines) remaining.set(line, (remaining.get(line) ?? 0) + 1); + const added: string[] = []; + for (const line of newLines) { + const count = remaining.get(line) ?? 0; + if (count === 0) added.push(line); + else remaining.set(line, count - 1); + } + const removed: string[] = []; + for (const [line, count] of remaining) { + for (let index = 0; index < count; index += 1) removed.push(line); + } + const url = + before.observation.url === after.observation.url + ? undefined + : { before: before.observation.url, after: after.observation.url }; + const title = + before.observation.title === after.observation.title + ? undefined + : { before: before.observation.title, after: after.observation.title }; + return { + changed: added.length > 0 || removed.length > 0 || url !== undefined || title !== undefined, + added, + removed, + ...(url ? { url } : {}), + ...(title ? { title } : {}), + }; +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + function tabOf(action: { tab_id?: string }): string | undefined { return action.tab_id; } @@ -954,6 +1550,10 @@ function cohortKey(role: string, name: string): string { return `${role}\u0000${name}`; } +function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { + return `${parentFrameKey}\u0000${backendNodeId}`; +} + /** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { let end = start; diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 5610d31b..2e49cdc6 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -134,7 +134,9 @@ export class InternalComputerTranslator { for (const action of actions) { if (isCuaBrowserAction(action)) { await flush(); - result.readResults.push(...(await this.browser().execute(action))); + const reads = await this.browser().execute(action); + result.readResults.push(...reads); + if (action.type === "browser_act" && reads.some((read) => read.type === "browser_act" && read.result.stop_reason)) break; continue; } switch (action.type) { diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 55f793d7..633ed1c2 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -1,8 +1,59 @@ +export type BrowserActOutcome = "worked" | "didnt" | "unknown"; +export type BrowserExpectationStatus = "newly_verified" | "preexisting" | "failed" | "unverifiable"; + +export interface BrowserExpectationEvidence { + status: BrowserExpectationStatus; + before?: boolean; + after?: boolean; + details: string[]; +} + +export interface BrowserActStepResult { + index: number; + type: string; + outcome: BrowserActOutcome; + evidence: string[]; + expectation?: BrowserExpectationEvidence; +} + +export interface BrowserObservationDiff { + changed: boolean; + added: string[]; + removed: string[]; + url?: { before: string; after: string }; + title?: { before: string; after: string }; +} + +export interface BrowserActObservedSuccessor { + status: "observed"; + text: string; + url: string; + title: string; + diff: BrowserObservationDiff; +} + +export interface BrowserActUnavailableSuccessor { + status: "unavailable"; + error: string; +} + +export type BrowserActSuccessor = BrowserActObservedSuccessor | BrowserActUnavailableSuccessor; + +export interface BrowserActResult { + outcome: BrowserActOutcome; + steps: BrowserActStepResult[]; + stopped_at?: number; + stop_reason?: "action_failed" | "expectation_failed" | "navigation" | "stale_ref" | "dialog" | "control_flow"; + final_expectation?: BrowserExpectationEvidence; + successor: BrowserActSuccessor; +} + export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } - | { type: "browser_text"; label: string; text: string }; + | { type: "browser_text"; label: string; text: string } + | { type: "browser_act"; result: BrowserActResult }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index b345b62f..f004d91d 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1,7 +1,7 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; -import type { CuaBrowserAction } from "@onkernel/cua-ai"; +import { computerToolExecutors, type CuaBrowserAction } from "@onkernel/cua-ai"; import { BrowserExecutor } from "../src/translator/browser"; import type { CdpConnection } from "../src/translator/cdp"; import { buildCuaComputerTools } from "../src/tools"; @@ -32,6 +32,18 @@ function createFakeBrowserExecutor() { execute: async (action: CuaBrowserAction): Promise => { executed.push(action); if (action.type === "browser_text") return [{ type: "browser_text", label: "text", text: "hello" }]; + if (action.type === "browser_act") { + const stopped = action.steps[0]?.type === "click" && action.steps[0].ref === "stale"; + return [{ + type: "browser_act", + result: { + outcome: stopped ? "didnt" : "unknown", + steps: [{ index: 0, type: "click", outcome: stopped ? "didnt" : "unknown", evidence: ["input delivered"] }], + ...(stopped ? { stopped_at: 0, stop_reason: "stale_ref" as const } : {}), + successor: { status: "observed", text: 'button "Save" [e2]', url: "https://a.test/", title: "Page", diff: { changed: false, added: [], removed: [] } }, + }, + }]; + } return []; }, screenshot: async () => ({ data: Buffer.from("png"), mimeType: "image/png" }), @@ -56,6 +68,35 @@ describe("InternalComputerTranslator browser plane", () => { expect(result.readResults).toEqual([{ type: "browser_text", label: "text", text: "hello" }]); }); + it("surfaces browser_act as readable content and structured tool details without a fallback screenshot", async () => { + const { client } = createClient(); + const { executor } = createFakeBrowserExecutor(); + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); + const actExecutor = computerToolExecutors({ mode: "browser" }).find((candidate) => candidate.definition.name === "act")!; + const tool = buildCuaComputerTools({ toolExecutors: [actExecutor], mode: "browser" }, translator).find((candidate) => candidate.name === "act")!; + + const result = await tool.execute("call_1", { steps: [{ type: "click", ref: "e1" }] }); + + expect(result.content).toHaveLength(1); + expect(result.content[0]).toMatchObject({ type: "text" }); + expect((result.content[0] as { text: string }).text).toContain("browser_act outcome: unknown"); + expect(result.details.statusText).toBe("Browser action outcome is unknown."); + expect(result.details.readResults[0]).toMatchObject({ type: "browser_act", result: { outcome: "unknown" } }); + }); + + it("stops a mixed batch after browser_act reaches a stop boundary", async () => { + const { client } = createClient(); + const { executor, executed } = createFakeBrowserExecutor(); + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); + + await translator.executeBatch([ + { type: "browser_act", steps: [{ type: "click", ref: "stale" }] }, + { type: "browser_type", text: "must not run" }, + ]); + + expect(executed.map((action) => action.type)).toEqual(["browser_act"]); + }); + it("errors on browser actions when the browser has no cdp_ws_url", async () => { const { client } = createClient(); const translator = new InternalComputerTranslator({ browser: { session_id: "b" } as KernelBrowser, client }); @@ -106,6 +147,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { const listeners: Array<(event: FakeCdpEvent) => void> = []; let nodes = initialNodes as Array<{ backendDOMNodeId?: number }>; let cursorBackendIds: number[] = []; + let targets = [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }]; const sessionTrees = new Map>(); const frameTrees = new Map>(); const iframeFrameIds = new Map(); @@ -120,7 +162,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { return nodes; }; const requireBackendId = (id: unknown, sessionId?: string) => { - const candidates = sessionId && sessionTrees.has(sessionId) ? [sessionTrees.get(sessionId)!] : [nodes, ...frameTrees.values()]; + const candidates = sessionId && sessionTrees.has(sessionId) + ? [sessionTrees.get(sessionId)!, ...frameTrees.values()] + : [nodes, ...frameTrees.values()]; if (!candidates.some((tree) => tree.some((node) => node.backendDOMNodeId === id))) throw new Error("No node with given id found"); }; const fake = { @@ -170,7 +214,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { return {}; } }, - pageTargets: async () => [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], + pageTargets: async () => targets, attachToTarget: async () => "session-1", createTarget: async () => "TARGET-2", close: () => {}, @@ -178,6 +222,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { const setNodes = (next: unknown[]) => { nodes = next as Array<{ backendDOMNodeId?: number }>; }; + const setTargets = (next: typeof targets) => { + targets = next; + }; const setCursorBackendIds = (ids: number[]) => { cursorBackendIds = ids; }; @@ -200,6 +247,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { sent, emit, setNodes, + setTargets, setCursorBackendIds, setSessionTree, setFrameTree, @@ -212,6 +260,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { interface AXNodeSpec { nodeId: string; + ignored?: boolean; role?: string; name?: string; value?: unknown; @@ -224,6 +273,7 @@ interface AXNodeSpec { function ax(spec: AXNodeSpec) { return { nodeId: spec.nodeId, + ignored: spec.ignored, parentId: spec.parentId, childIds: spec.childIds, backendDOMNodeId: spec.backendDOMNodeId, @@ -647,6 +697,722 @@ describe("BrowserExecutor snapshot diffing", () => { await snapshotText(executor); expect(await snapshotText(executor, { filter: "interactive" })).toContain('button "Save" [e2]'); }); + + it("retries when an earlier iframe navigates while later frames are collected", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 51, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-1"); + fake.setFrameTree("FRAME-1", [ax({ nodeId: "f1", role: "RootWebArea", name: "Old frame" })]); + fake.setIframeFrame(51, "FRAME-2"); + fake.setFrameTree("FRAME-2", [ax({ nodeId: "g1", role: "RootWebArea", name: "Other frame" })]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let changed = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Accessibility.getFullAXTree" && params?.frameId === "FRAME-2" && !changed) { + changed = true; + fake.setFrameTree("FRAME-1", [ax({ nodeId: "f1", role: "RootWebArea", name: "Updated frame" })]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-1", parentId: "TARGET-1" } }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + + const text = await snapshotText(new BrowserExecutor(wrapped)); + expect(text).toContain('RootWebArea "Updated frame"'); + expect(text).not.toContain('RootWebArea "Old frame"'); + }); + + it("generation-fences browser_find before minting refs", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let changed = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Accessibility.getFullAXTree" && !changed) { + changed = true; + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Delete", backendDOMNodeId: 43, parentId: "1" }), + ]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + + const results = await executor.execute({ type: "browser_find", query: "delete button" } as CuaBrowserAction); + const read = results[0]; + if (!read || read.type !== "browser_text") throw new Error("expected find result"); + expect(read.text).toContain('button "Delete"'); + expect(read.text).not.toContain('button "Save"'); + }); + + it("retries when navigation changes the document during observation", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let changed = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Accessibility.getFullAXTree" && !changed) { + changed = true; + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Delete", backendDOMNodeId: 43, parentId: "1" }), + ]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + + const text = await snapshotText(new BrowserExecutor(wrapped)); + expect(text).toContain('button "Delete"'); + expect(text).not.toContain('button "Save"'); + }); +}); + +describe("BrowserExecutor browser_act", () => { + const actResult = async (executor: BrowserExecutor, action: Omit, "type">) => { + const reads = await executor.execute({ type: "browser_act", ...action }); + const read = reads.find((item) => item.type === "browser_act"); + if (!read || read.type !== "browser_act") throw new Error("expected browser_act result"); + return read.result; + }; + const observedSuccessor = (result: Awaited>) => { + if (result.successor.status !== "observed") throw new Error("expected observed successor"); + return result.successor; + }; + + it("reports a newly verified expectation and a structured successor diff", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" }), + ]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [ + { + type: "click", + ref: "e1", + expect: { + any: [ + { type: "role_name", role: "StaticText", name: "Saved" }, + { type: "text", text: "missing" }, + ], + }, + }, + ], + }); + + expect(result.outcome).toBe("worked"); + expect(result.steps[0]?.expectation).toMatchObject({ status: "newly_verified", before: false, after: true }); + expect(observedSuccessor(result).diff).toMatchObject({ changed: true, added: [' StaticText "Saved"'] }); + expect(observedSuccessor(result).text).toContain('StaticText "Saved"'); + expect(await snapshotText(executor)).toBe("Page unchanged since the last snapshot; previous element refs are still valid."); + }); + + it("reports a failed postcondition and does not execute later steps", async () => { + const { cdp, sent } = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Next", backendDOMNodeId: 43, parentId: "1" }), + ]); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [ + { type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }, + { type: "click", ref: "e2" }, + ], + }); + + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "expectation_failed" }); + expect(result.steps[0]?.expectation?.status).toBe("failed"); + expect(sent.filter((command) => command.method === "Input.dispatchMouseEvent" && command.params.type === "mousePressed")).toHaveLength(1); + }); + + it("labels an already-satisfied expectation as preexisting instead of success", async () => { + const { cdp } = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" }), + ]); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }], + }); + expect(result.outcome).toBe("unknown"); + expect(result.steps[0]?.expectation?.status).toBe("preexisting"); + }); + + it("requires a supplied final expectation to be newly verified", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([...BUTTON_TREE, ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" })]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }], + expect: { type: "title", equals: "Page" }, + }); + + expect(result.steps[0]?.expectation?.status).toBe("newly_verified"); + expect(result.final_expectation?.status).toBe("preexisting"); + expect(result.outcome).toBe("unknown"); + }); + + it("stops a dependent list when navigation invalidates its refs", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Go", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Later", backendDOMNodeId: 43, parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "click", ref: "e2" }] }); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "navigation" }); + expect(result.steps).toHaveLength(1); + }); + + it("keeps a navigation expectation verified while stopping later dispatch", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setTargets([{ targetId: "TARGET-1", type: "page", title: "Next", url: "https://a.test/next" }]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "url", contains: "/next", changed: true } }], + }); + + expect(result).toMatchObject({ outcome: "worked", stopped_at: 0, stop_reason: "navigation" }); + expect(result.steps[0]).toMatchObject({ outcome: "worked", expectation: { status: "newly_verified" } }); + }); + + it("does not treat a final step skipped by navigation as completed", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { + send: (method: string, params?: Record, sessionId?: string) => Promise; + pageTargets: () => Promise>; + }; + let released = false; + let targetReadsAfterRelease = 0; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") released = true; + return result; + }, + pageTargets: async () => { + const targets = await inner.pageTargets(); + if (released && ++targetReadsAfterRelease === 8) { + fake.setTargets([{ targetId: "TARGET-1", type: "page", title: "Next", url: "https://a.test/next" }]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return targets; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }, { type: "type", text: "must not run" }], + expect: { type: "url", contains: "/next", changed: true }, + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 1, stop_reason: "navigation" }); + expect(result.steps[1]).toMatchObject({ outcome: "unknown", evidence: ["navigation detected before input delivery"] }); + expect(fake.sent.some((command) => command.method === "Input.insertText")).toBe(false); + }); + + it("stops after a dialog changes control flow", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Delete", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Later", backendDOMNodeId: 43, parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.emit({ method: "Page.javascriptDialogOpening", params: { type: "confirm", message: "Delete?" }, sessionId: "session-1" }); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "click", ref: "e2" }] }); + expect(result).toMatchObject({ stopped_at: 0, stop_reason: "dialog" }); + expect(result.steps).toHaveLength(1); + }); + + it("polls delayed postconditions and uses a fresh observation before each action", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let released = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased" && !released) { + released = true; + setTimeout(() => fake.setNodes([ + ...BUTTON_TREE, + ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" }), + ]), 30); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [ + { type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }, + { type: "wait", ms: 0, expect: { type: "text", text: "Saved" } }, + ], + }); + expect(result.outcome).toBe("unknown"); + expect(result.steps.map((step) => step.expectation?.status)).toEqual(["newly_verified", "preexisting"]); + }); + + it("does not satisfy accessible expectations from ignored nodes", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([...BUTTON_TREE, ax({ nodeId: "3", ignored: true, role: "StaticText", name: "Saved", parentId: "1" })]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const textResult = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }], + }); + expect(textResult.steps[0]?.expectation?.status).toBe("failed"); + + const roleResult = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "role_name", role: "StaticText", name: "Saved" } }], + }); + expect(roleResult.steps[0]?.expectation?.status).toBe("failed"); + }); + + it("stops when a post-action observation is unavailable", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let failObservation = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + if (method === "Accessibility.getFullAXTree" && failObservation) throw new Error("document replaced"); + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") failObservation = true; + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }, { type: "type", text: "later" }], + }); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "control_flow" }); + expect(result.steps[0]?.expectation?.status).toBe("unverifiable"); + expect(result.successor.status).toBe("unavailable"); + expect(fake.sent.some((command) => command.method === "Input.insertText")).toBe(false); + }); + + it("returns an observed successor when dispatch fails after changing the page", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "Partially saved", parentId: "1" }), + ]); + throw new Error("mouse release acknowledgement lost"); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Partially saved" } }], + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "action_failed" }); + expect(result.steps[0]).toMatchObject({ outcome: "unknown", expectation: { status: "newly_verified" } }); + expect(observedSuccessor(result).text).toContain("Partially saved"); + expect(observedSuccessor(result).diff.added).toContain(' StaticText "Partially saved"'); + }); + + it("detects delayed navigation while polling a final expectation", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let released = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased" && !released) { + released = true; + setTimeout(() => fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }), 20); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }], + expect: { type: "text", text: "Never appears" }, + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 1, stop_reason: "navigation" }); + }); + + it("recollects the successor when navigation lands during the final boundary check", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { + send: (method: string, params?: Record, sessionId?: string) => Promise; + pageTargets: () => Promise>; + }; + let released = false; + let targetReadsAfterRelease = 0; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") released = true; + return result; + }, + pageTargets: async () => { + const targets = await inner.pageTargets(); + if (released && ++targetReadsAfterRelease === 8) { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "New page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Continue", backendDOMNodeId: 84, parentId: "1" }), + ]); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + } + return targets; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }] }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 1, stop_reason: "navigation" }); + expect(observedSuccessor(result).text).toContain('button "Continue"'); + }); + + it("detects delayed navigation and ignores unrelated target generations", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Go", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Later", backendDOMNodeId: 43, parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let released = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased" && !released) { + released = true; + setTimeout(() => fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }), 0); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + (executor as unknown as { generations: Map }).generations.set("OTHER-TARGET", 1); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "click", ref: "e2" }] }); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "navigation" }); + expect(result.steps).toHaveLength(1); + }); + + it("stops when an action opens a new page target", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setTargets([ + { targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }, + { targetId: "TARGET-2", type: "page", title: "Popup", url: "https://b.test/" }, + ]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "later" }] }); + expect(result).toMatchObject({ stopped_at: 0, stop_reason: "control_flow" }); + expect(result.steps).toHaveLength(1); + }); + + it("keeps the successor diff complete when text presentation is filtered", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" }), + ]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }], successor: { filter: "interactive" } }); + expect(observedSuccessor(result).text).not.toContain("Saved"); + expect(observedSuccessor(result).diff.added).toContain(' StaticText "Saved"'); + }); + + it("conservatively heals ref expectations after an unambiguous rerender", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "checkbox", name: "Ready", properties: [{ name: "checked", value: false }], backendDOMNodeId: 42, parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["9"] }), + ax({ nodeId: "9", role: "checkbox", name: "Ready", properties: [{ name: "checked", value: true }], backendDOMNodeId: 99, parentId: "1" }), + ]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "ref", ref: "e1", checked: true } }], + }); + expect(result.outcome).toBe("worked"); + expect(result.steps[0]?.expectation?.status).toBe("newly_verified"); + }); + + it("evaluates ref value/state and final URL, title, and disappearance expectations", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ + nodeId: "2", + role: "option", + name: "Choice", + value: "old", + properties: [{ name: "selected", value: false }, { name: "expanded", value: false }], + backendDOMNodeId: 42, + parentId: "1", + }), + ax({ nodeId: "3", role: "StaticText", name: "Loading", parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Done", childIds: ["2"] }), + ax({ + nodeId: "2", + role: "option", + name: "Choice", + value: "new", + properties: [{ name: "selected", value: true }, { name: "expanded", value: true }], + backendDOMNodeId: 42, + parentId: "1", + }), + ]); + fake.setTargets([{ targetId: "TARGET-1", type: "page", title: "Done", url: "https://a.test/saved" }]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ + type: "click", + ref: "e1", + expect: { all: [ + { type: "ref", ref: "e1", value: "new", selected: true, expanded: true }, + { type: "text", text: "Loading", exists: false }, + ] }, + }], + expect: { all: [ + { type: "url", contains: "/saved", changed: true }, + { type: "title", equals: "Done", changed: true }, + ] }, + }); + expect(result.outcome).toBe("worked"); + expect(result.steps[0]?.expectation?.status).toBe("newly_verified"); + expect(result.final_expectation?.status).toBe("newly_verified"); + expect(observedSuccessor(result).diff.url?.after).toBe("https://a.test/saved"); + }); + + it("does not verify absence against an incomplete nested iframe observation", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-1"); + fake.setFrameTree("FRAME-1", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "First frame", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "Iframe", backendDOMNodeId: 60, parentId: "f1" }), + ]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Missing", exists: false } }], + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "control_flow" }); + expect(result.steps[0]?.expectation?.status).toBe("unverifiable"); + }); + + it("keeps ambiguous ref replacement unverifiable and stops", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "checkbox", name: "Ready", backendDOMNodeId: 42, parentId: "1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["8", "9"] }), + ax({ nodeId: "8", role: "checkbox", name: "Ready", backendDOMNodeId: 98, parentId: "1" }), + ax({ nodeId: "9", role: "checkbox", name: "Ready", properties: [{ name: "checked", value: true }], backendDOMNodeId: 99, parentId: "1" }), + ]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "ref", ref: "e1", checked: true } }, { type: "type", text: "later" }], + }); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "control_flow" }); + expect(result.steps[0]?.expectation?.status).toBe("unverifiable"); + }); + + it("sets the final expectation stop boundary after all actions", async () => { + const executor = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); + await snapshotText(executor); + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }], + expect: { type: "title", equals: "Missing" }, + }); + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 1, stop_reason: "expectation_failed" }); + expect(result.final_expectation?.status).toBe("failed"); + }); + + it("returns a stale-ref outcome instead of dispatching later actions", async () => { + const { cdp, emit, sent } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "later" }] }); + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "stale_ref" }); + expect(sent.some((command) => command.method === "Input.insertText")).toBe(false); + }); }); describe("BrowserExecutor iframe stitching", () => { @@ -687,6 +1453,106 @@ describe("BrowserExecutor iframe stitching", () => { return fake; }; + it("recursively stitches nested iframe subtrees and invalidates descendants with their parent", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-1"); + fake.setFrameTree("FRAME-1", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "First", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "Iframe", backendDOMNodeId: 60, parentId: "f1" }), + ]); + fake.setIframeFrame(60, "FRAME-2"); + fake.setFrameTree("FRAME-2", [ + ax({ nodeId: "g1", role: "RootWebArea", name: "Second", childIds: ["g2"] }), + ax({ nodeId: "g2", role: "button", name: "Deep", backendDOMNodeId: 70, parentId: "g1" }), + ]); + const executor = new BrowserExecutor(fake.cdp); + + const text = await snapshotText(executor); + + expect(text).toContain('button "Deep" [e3]'); + await executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + expect(fake.sent).toContainEqual(expect.objectContaining({ method: "DOM.getBoxModel", params: { backendNodeId: 70 } })); + + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-1", parentId: "TARGET-1" } }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + }); + + it("invalidates nested same-process refs when their OOPIF session navigates", async () => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Widget", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "Iframe", backendDOMNodeId: 60, parentId: "f1" }), + ]); + fake.setIframeFrame(60, "FRAME-INNER"); + fake.setFrameTree("FRAME-INNER", [ + ax({ nodeId: "g1", role: "RootWebArea", name: "Inner", childIds: ["g2"] }), + ax({ nodeId: "g2", role: "button", name: "Deep", backendDOMNodeId: 80, parentId: "g1" }), + ]); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Deep" [e4]'); + + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-INNER", parentId: "FRAME-OOP" } }, sessionId: "session-oop" }); + + await expect(executor.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + }); + + it("rebinds imported nested OOPIF refs only through the owning OOPIF session", async () => { + const fake = createFakeCdp(OOPIF_PAGE); + const configureAutoAttach = () => fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setIframeFrame(50, "FRAME-OOP"); + configureAutoAttach(); + fake.setSessionTree("session-oop", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Widget", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "Iframe", backendDOMNodeId: 60, parentId: "f1" }), + ]); + fake.setIframeFrame(60, "FRAME-INNER"); + fake.setFrameTree("FRAME-INNER", [ + ax({ nodeId: "g1", role: "RootWebArea", name: "Inner", childIds: ["g2"] }), + ax({ nodeId: "g2", role: "button", name: "Deep", backendDOMNodeId: 80, parentId: "g1" }), + ]); + const first = new BrowserExecutor(fake.cdp); + const text = await snapshotText(first); + expect(text).toContain('button "Deep" [e4]'); + const state = first.exportRefState(); + first.close(); + + configureAutoAttach(); + const resumed = new BrowserExecutor(fake.cdp); + resumed.importRefState(state); + await resumed.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction); + + const resolved = fake.sent.filter((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 80).at(-1); + expect(resolved?.sessionId).toBe("session-oop"); + + const resolvedCount = fake.sent.filter((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 80).length; + const unavailable = new BrowserExecutor(fake.cdp); + unavailable.importRefState(state); + await expect(unavailable.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction)).rejects.toThrow(/owning frame session/); + expect(fake.sent.filter((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 80)).toHaveLength(resolvedCount); + + configureAutoAttach(); + const scrolling = new BrowserExecutor(fake.cdp); + scrolling.importRefState(state); + await scrolling.execute({ type: "browser_scroll_to", ref: "e4" } as CuaBrowserAction); + const scrolled = fake.sent.filter((command) => command.method === "DOM.scrollIntoViewIfNeeded" && command.params.backendNodeId === 80).at(-1); + expect(scrolled?.sessionId).toBe("session-oop"); + + configureAutoAttach(); + const ancestry = new BrowserExecutor(fake.cdp); + ancestry.importRefState(state); + await ancestry.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-OOP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + await expect(ancestry.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + it("resolves an OOPIF ref's node through the child session but dispatches input on the page session", async () => { const { cdp, sent } = setupOopif(); const executor = new BrowserExecutor(cdp); diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index 55589ac6..c37c80c7 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -17,6 +17,7 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; */ export const CUA_BROWSER_ACTION_TYPES = [ "browser_snapshot", + "browser_act", "browser_text", "browser_find", "browser_click", @@ -44,6 +45,62 @@ export interface CuaActionBrowserSnapshot { tab_id?: string; } +type RoleNameExpectation = + | { type: "role_name"; role: string; name?: string; exists?: boolean } + | { type: "role_name"; role?: string; name: string; exists?: boolean }; + +type RefExpectationState = { + value?: string; + checked?: boolean | "mixed"; + selected?: boolean; + expanded?: boolean; +}; + +type RefExpectation = { type: "ref"; ref: string } & ( + | (RefExpectationState & { value: string }) + | (RefExpectationState & { checked: boolean | "mixed" }) + | (RefExpectationState & { selected: boolean }) + | (RefExpectationState & { expanded: boolean }) +); + +type LocationExpectation = { type: "url" | "title" } & ( + | { equals: string; contains?: string; changed?: boolean } + | { equals?: string; contains: string; changed?: boolean } + | { equals?: string; contains?: string; changed: boolean } +); + +type CuaBrowserExpectationLeaf = + | { type: "text"; text: string; exists?: boolean } + | RoleNameExpectation + | RefExpectation + | LocationExpectation; + +type NonEmptyArray = [T, ...T[]]; + +export type CuaBrowserExpectation = + | CuaBrowserExpectationLeaf + | { all: NonEmptyArray } + | { any: NonEmptyArray }; + +export type CuaBrowserActStep = + | { type: "click"; ref: string; button?: "left" | "right" | "middle"; num_clicks?: number; modifiers?: string[]; expect?: CuaBrowserExpectation } + | { type: "hover"; ref: string; expect?: CuaBrowserExpectation } + | { type: "fill"; ref: string; value: string | number | boolean; expect?: CuaBrowserExpectation } + | { type: "type"; text: string; expect?: CuaBrowserExpectation } + | { type: "key"; text: string; repeat?: number; expect?: CuaBrowserExpectation } + | { type: "scroll_to"; ref: string; expect?: CuaBrowserExpectation } + | { type: "wait"; ms?: number; expect?: CuaBrowserExpectation }; + +export interface CuaActionBrowserAct { + type: "browser_act"; + steps: NonEmptyArray; + /** Optional final semantic expectation, evaluated against the initial and successor observations. */ + expect?: CuaBrowserExpectation; + /** Controls presentation of the single successor observation. */ + successor?: { filter?: "all" | "interactive"; depth?: number }; + tab_id?: string; +} + export interface CuaActionBrowserText { type: "browser_text"; tab_id?: string; @@ -146,6 +203,7 @@ export interface CuaActionBrowserEvaluate { export type CuaBrowserAction = | CuaActionBrowserSnapshot + | CuaActionBrowserAct | CuaActionBrowserText | CuaActionBrowserFind | CuaActionBrowserClick @@ -179,6 +237,79 @@ const TabId = () => Type.Optional(Type.String({ description: "Tab to act on. Def const RefProperty = () => Type.String({ description: "Element reference from browser_snapshot or browser_find, e.g. \"e12\"." }); export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOptions): Record { + const expectationLeaves = (): TSchema[] => { + const roleName = (required: "role" | "name") => + Type.Object( + { + type: Type.Literal("role_name"), + role: required === "role" ? Type.String() : Type.Optional(Type.String()), + name: required === "name" ? Type.String() : Type.Optional(Type.String()), + exists: Type.Optional(Type.Boolean()), + }, + { additionalProperties: false }, + ); + const refState = (required: "value" | "checked" | "selected" | "expanded") => + Type.Object( + { + type: Type.Literal("ref"), + ref: RefProperty(), + value: required === "value" ? Type.String() : Type.Optional(Type.String()), + checked: + required === "checked" + ? Type.Union([Type.Boolean(), Type.Literal("mixed")]) + : Type.Optional(Type.Union([Type.Boolean(), Type.Literal("mixed")])), + selected: required === "selected" ? Type.Boolean() : Type.Optional(Type.Boolean()), + expanded: required === "expanded" ? Type.Boolean() : Type.Optional(Type.Boolean()), + }, + { additionalProperties: false }, + ); + const location = (required: "equals" | "contains" | "changed") => + Type.Object( + { + type: Type.Union([Type.Literal("url"), Type.Literal("title")]), + equals: required === "equals" ? Type.String() : Type.Optional(Type.String()), + contains: required === "contains" ? Type.String() : Type.Optional(Type.String()), + changed: required === "changed" ? Type.Boolean() : Type.Optional(Type.Boolean()), + }, + { additionalProperties: false }, + ); + return [ + Type.Object({ type: Type.Literal("text"), text: Type.String(), exists: Type.Optional(Type.Boolean()) }, { additionalProperties: false }), + roleName("role"), + roleName("name"), + refState("value"), + refState("checked"), + refState("selected"), + refState("expanded"), + location("equals"), + location("contains"), + location("changed"), + ]; + }; + const expectationLeaf = Type.Union(expectationLeaves(), { $id: "BrowserExpectationLeaf" }); + const expectationLeafRef = () => Type.Ref("BrowserExpectationLeaf"); + const expectation = Type.Union( + [ + expectationLeafRef(), + Type.Object({ all: Type.Array(expectationLeafRef(), { minItems: 1 }) }, { additionalProperties: false }), + Type.Object({ any: Type.Array(expectationLeafRef(), { minItems: 1 }) }, { additionalProperties: false }), + ], + { $id: "BrowserExpectation" }, + ); + const expectationRef = () => Type.Ref("BrowserExpectation"); + const stepExpectation = { expect: Type.Optional(expectationRef()) }; + const actStep = Type.Union( + [ + Type.Object({ type: Type.Literal("click"), ref: RefProperty(), button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])), num_clicks: Type.Optional(Type.Number()), modifiers: Type.Optional(Type.Array(Type.String())), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("hover"), ref: RefProperty(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("fill"), ref: RefProperty(), value: Type.Union([Type.String(), Type.Number(), Type.Boolean()]), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("type"), text: Type.String(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("key"), text: Type.String(), repeat: Type.Optional(Type.Number()), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("scroll_to"), ref: RefProperty(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("wait"), ms: Type.Optional(Type.Number({ minimum: 0, maximum: 30_000 })), ...stepExpectation }, { additionalProperties: false }), + ], + { $id: "BrowserActStep" }, + ); const clickTarget: Record = options.coordinates ? { ref: Type.Optional(RefProperty()), @@ -198,6 +329,31 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), + browser_act: Type.Object( + { + type: Type.Literal("browser_act"), + steps: Type.Array(Type.Ref("BrowserActStep"), { minItems: 1, maxItems: 20 }), + expect: Type.Optional(expectationRef()), + successor: Type.Optional( + Type.Object( + { + filter: Type.Optional(Type.Union([Type.Literal("all"), Type.Literal("interactive")])), + depth: Type.Optional(Type.Number()), + }, + { additionalProperties: false }, + ), + ), + tab_id: TabId(), + }, + { + additionalProperties: false, + $defs: { + BrowserExpectationLeaf: expectationLeaf, + BrowserExpectation: expectation, + BrowserActStep: actStep, + }, + }, + ), browser_text: Type.Object( { type: Type.Literal("browser_text"), diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index cbc83f7d..5e7ab47a 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -49,13 +49,15 @@ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] /** * Browser actions exposed in hybrid mode: reads, element-targeted writes, - * and JavaScript evaluation. Pointer/keyboard capabilities (`browser_click` by coordinate, - * `browser_type`, `browser_key`, `browser_scroll`, `browser_hover`, `browser_drag`) and + * dependent ref/focus action lists, and JavaScript evaluation. Standalone + * pointer/keyboard capabilities (`browser_click` by coordinate, `browser_type`, + * `browser_key`, `browser_scroll`, `browser_hover`, `browser_drag`) and * `browser_screenshot` are excluded — real OS input and the OS screenshot cover - * those, keeping one tool per capability and one coordinate frame. + * those, keeping one coordinate frame. */ export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = [ "browser_snapshot", + "browser_act", "browser_text", "browser_find", "browser_click", @@ -113,6 +115,8 @@ const BROWSER_ACTION_DESCRIPTIONS: Record = { "Return an accessibility-tree snapshot of the page, including iframe content, with element references like [e12]. " + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale. " + "If the page has not changed since your previous snapshot, a short unchanged notice is returned instead and earlier refs remain valid.", + browser_act: + "Execute one dependent list of ref/focus-based browser actions and return honest semantic outcomes, per-step evidence, stop boundaries, and one successor observation diff. Input delivery alone is not reported as success.", browser_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", browser_find: "Find elements on the page (including iframe content) matching a natural-language description and return them with element references, like a filtered snapshot.", diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index dc54a620..90463211 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -40,8 +40,9 @@ type ObjectSchemaWithProperties = TSchema & { properties: Record }; + const { type: _type, ...properties } = schema.properties; + return Type.Object(properties, { additionalProperties: false, ...(schema.$defs ? { $defs: schema.$defs } : {}) }); } export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): TSchema { diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index d9b2f030..6f8cb92e 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -1,3 +1,4 @@ +import { validateToolArguments } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; import { CUA_ACTION_TYPES, @@ -6,6 +7,7 @@ import { CUA_HYBRID_COMPUTER_ACTION_TYPES, anthropic, computerTools, + createCuaBatchSchema, cuaToolNameForAction, defaultActionsForMode, openai, @@ -20,6 +22,7 @@ describe("mode action sets", () => { it("browser mode defaults to every browser action plus wait", () => { const actions = defaultActionsForMode("browser"); expect(actions).toContain("browser_snapshot"); + expect(actions).toContain("browser_act"); expect(actions).toContain("wait"); expect(actions).toContain("browser_evaluate"); expect(actions).not.toContain("click"); @@ -31,6 +34,7 @@ describe("mode action sets", () => { expect(actions).not.toContain("goto"); expect(actions).not.toContain("url"); expect(actions).toContain("browser_navigate"); + expect(actions).toContain("browser_act"); // One screenshot: the OS display. expect(actions).toContain("screenshot"); expect(actions).toContain("zoom"); @@ -50,12 +54,14 @@ describe("mode tool naming", () => { it("browser mode strips the browser_ prefix", () => { expect(cuaToolNameForAction("browser_snapshot", "browser")).toBe("snapshot"); expect(cuaToolNameForAction("browser_click", "browser")).toBe("click"); + expect(cuaToolNameForAction("browser_act", "browser")).toBe("act"); expect(cuaToolNameForAction("wait", "browser")).toBe("wait"); }); it("hybrid mode prefixes computer actions and keeps browser_ names", () => { expect(cuaToolNameForAction("click", "hybrid")).toBe("computer_click"); expect(cuaToolNameForAction("browser_click", "hybrid")).toBe("browser_click"); + expect(cuaToolNameForAction("browser_act", "hybrid")).toBe("browser_act"); }); it("computer mode rejects browser actions", () => { @@ -79,6 +85,60 @@ describe("mode tool schemas", () => { expect(pageClick.parameters.required).toContain("ref"); }); + it("exposes browser_act as a ref-only dependent action schema", () => { + const browserAct = computerTools({ mode: "browser" }).find((tool) => tool.name === "act")!; + const hybridAct = computerTools({ mode: "hybrid" }).find((tool) => tool.name === "browser_act")!; + for (const tool of [browserAct, hybridAct]) { + expect(tool.parameters.properties.steps).toBeDefined(); + expect(tool.parameters.properties.expect).toBeDefined(); + expect(tool.parameters.properties.successor).toBeDefined(); + expect(JSON.stringify(tool.parameters.properties.steps)).not.toContain('"x"'); + expect(JSON.stringify(tool.parameters).length).toBeLessThan(10_000); + } + }); + + it("keeps browser_act definitions resolvable inside a batch schema", () => { + const batch = { + name: "batch", + description: "batch", + parameters: createCuaBatchSchema(defaultActionsForMode("browser"), "browser"), + }; + expect(() => + validateToolArguments(batch, { + id: "call_1", + name: "batch", + arguments: { + actions: [{ type: "browser_act", steps: [{ type: "wait", ms: 0, expect: { type: "url", changed: true } }] }], + }, + }), + ).not.toThrow(); + }); + + it("rejects semantic leaves without a predicate", () => { + const act = computerTools({ mode: "browser" }).find((tool) => tool.name === "act")!; + for (const predicate of [ + { type: "role_name" }, + { type: "ref", ref: "e1" }, + { type: "url" }, + { type: "title" }, + ]) { + expect(() => + validateToolArguments(act, { + id: "call_1", + name: "act", + arguments: { steps: [{ type: "wait", ms: 0, expect: predicate }] }, + }), + ).toThrow(/Validation failed/); + } + expect(() => + validateToolArguments(act, { + id: "call_2", + name: "act", + arguments: { steps: [{ type: "wait", ms: 0, expect: { type: "url", changed: true } }] }, + }), + ).not.toThrow(); + }); + it("browser mode exposes every default browser action under its unprefixed name", () => { const tools = computerTools({ mode: "browser" }); const names = tools.map((tool) => tool.name); From 2c3eb9781d15ec4b9fe73ae4900fac595c89c286 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:09:15 +0000 Subject: [PATCH 02/18] Address browser action review feedback --- packages/agent/src/translator/browser.ts | 44 +++++++----- packages/agent/src/translator/types.ts | 10 +++ .../agent/test/translator-browser.test.ts | 70 +++++++++++++++++++ 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 7ce3e857..07189ff2 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -673,9 +673,13 @@ export class BrowserExecutor { break; } } - if (!stopReason && finalExpectation?.status === "failed") stopReason = "expectation_failed"; - else if (!stopReason && finalExpectation?.status === "unverifiable") stopReason = "control_flow"; - if (stopReason) stoppedAt = action.steps.length; + if (finalExpectation?.status === "failed") { + stopReason = "expectation_failed"; + stoppedAt = action.steps.length; + } else if (finalExpectation?.status === "unverifiable") { + stopReason ??= "control_flow"; + stoppedAt = action.steps.length; + } } let successor: BrowserActResult["successor"] | undefined; @@ -1294,13 +1298,16 @@ export class BrowserExecutor { */ private async refSession(entry: RefEntry): Promise { if (!entry.sessionId) { - await this.attach(entry.targetId); - const sessionTargetId = entry.sessionTargetId ?? entry.frameId; - const frameSession = this.frameSessions.get(sessionTargetId); - if (sessionTargetId !== entry.targetId && !frameSession) { + const pageSession = await this.attach(entry.targetId); + if (entry.sessionTargetId === undefined) { + entry.sessionId = this.frameSessions.get(entry.frameId) ?? pageSession; + return entry.sessionId; + } + const frameSession = this.frameSessions.get(entry.sessionTargetId); + if (entry.sessionTargetId !== entry.targetId && !frameSession) { throw new Error("owning frame session is unavailable; the element ref is stale"); } - entry.sessionId = frameSession ?? (await this.attach(entry.targetId)); + entry.sessionId = frameSession ?? pageSession; } return entry.sessionId; } @@ -1320,7 +1327,7 @@ export class BrowserExecutor { } private invalidateRefs(targetId: string): void { - this.generations.set(targetId, this.generation(targetId) + 1); + this.invalidateFrame(targetId); for (const [ref, entry] of this.refs) { if (entry.targetId === targetId) this.refs.delete(ref); } @@ -1339,8 +1346,11 @@ export class BrowserExecutor { } } for (const invalidatedFrame of invalidated) { - this.generations.set(invalidatedFrame, this.generation(invalidatedFrame) + 1); - if (invalidatedFrame !== frameKey) this.frameParents.delete(invalidatedFrame); + if (invalidatedFrame === frameKey) this.generations.set(invalidatedFrame, this.generation(invalidatedFrame) + 1); + else { + this.generations.delete(invalidatedFrame); + this.frameParents.delete(invalidatedFrame); + } } for (const [ref, entry] of this.refs) { if (invalidated.has(entry.frameId)) this.refs.delete(ref); @@ -1457,12 +1467,12 @@ function observationGenerationsChanged( after: BrowserObservation, live: ReadonlyMap, ): boolean { - const keys = new Set([...before.generations.keys(), ...after.generations.keys()]); - return [...keys].some( - (key) => - before.generations.get(key) !== after.generations.get(key) || - (after.generations.has(key) && after.generations.get(key) !== (live.get(key) ?? 0)), - ); + for (const [key, generation] of after.generations) { + if (generation !== (live.get(key) ?? 0)) return true; + const previous = before.generations.get(key); + if (previous !== undefined && previous !== generation) return true; + } + return false; } function browserControlChange( diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 633ed1c2..3c5095e9 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -1,6 +1,10 @@ +/** Semantic result of an action or dependent action list; input delivery alone remains `unknown`. */ export type BrowserActOutcome = "worked" | "didnt" | "unknown"; + +/** Whether an expectation changed to true, was already true, stayed false, or could not be evaluated. */ export type BrowserExpectationStatus = "newly_verified" | "preexisting" | "failed" | "unverifiable"; +/** Before/after evidence collected while evaluating a semantic expectation. */ export interface BrowserExpectationEvidence { status: BrowserExpectationStatus; before?: boolean; @@ -8,6 +12,7 @@ export interface BrowserExpectationEvidence { details: string[]; } +/** Outcome and evidence for one requested `browser_act` step. */ export interface BrowserActStepResult { index: number; type: string; @@ -16,6 +21,7 @@ export interface BrowserActStepResult { expectation?: BrowserExpectationEvidence; } +/** Complete normalized accessibility-tree and page-metadata changes between two observations. */ export interface BrowserObservationDiff { changed: boolean; added: string[]; @@ -24,6 +30,7 @@ export interface BrowserObservationDiff { title?: { before: string; after: string }; } +/** Stable successor observation collected after action execution stops. */ export interface BrowserActObservedSuccessor { status: "observed"; text: string; @@ -32,13 +39,16 @@ export interface BrowserActObservedSuccessor { diff: BrowserObservationDiff; } +/** Indicates that no stable successor observation could be collected. */ export interface BrowserActUnavailableSuccessor { status: "unavailable"; error: string; } +/** Stable observed successor or an explicit observation failure. */ export type BrowserActSuccessor = BrowserActObservedSuccessor | BrowserActUnavailableSuccessor; +/** Structured result for a dependent browser action list and its optional semantic expectations. */ export interface BrowserActResult { outcome: BrowserActOutcome; steps: BrowserActStepResult[]; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index f004d91d..03363a8c 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -941,10 +941,12 @@ describe("BrowserExecutor browser_act", () => { const result = await actResult(executor, { steps: [{ type: "click", ref: "e1", expect: { type: "url", contains: "/next", changed: true } }], + expect: { type: "url", contains: "/next", changed: true }, }); expect(result).toMatchObject({ outcome: "worked", stopped_at: 0, stop_reason: "navigation" }); expect(result.steps[0]).toMatchObject({ outcome: "worked", expectation: { status: "newly_verified" } }); + expect(result.final_expectation?.status).toBe("newly_verified"); }); it("does not treat a final step skipped by navigation as completed", async () => { @@ -1208,6 +1210,37 @@ describe("BrowserExecutor browser_act", () => { expect(result.steps).toHaveLength(1); }); + it("continues when a lazy iframe appears without invalidating existing refs", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-LAZY"); + fake.setFrameTree("FRAME-LAZY", [ax({ nodeId: "f1", role: "RootWebArea", name: "Lazy frame" })]); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }, { type: "type", text: "continued" }], + }); + + expect(result.stop_reason).toBeUndefined(); + expect(result.steps).toHaveLength(2); + expect(fake.sent).toContainEqual(expect.objectContaining({ method: "Input.insertText", params: { text: "continued" } })); + }); + it("stops when an action opens a new page target", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; @@ -1479,6 +1512,10 @@ describe("BrowserExecutor iframe stitching", () => { fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-1", parentId: "TARGET-1" } }, sessionId: "session-1" }); await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + expect((executor as unknown as { frameParents: Map }).frameParents.size).toBe(0); + expect((executor as unknown as { generations: Map }).generations.has("FRAME-1")).toBe(false); }); it("invalidates nested same-process refs when their OOPIF session navigates", async () => { @@ -1680,6 +1717,39 @@ describe("BrowserExecutor ref state export/import", () => { expect(pressed).toBeDefined(); }); + it("rebinds legacy same-process iframe refs to the page session", async () => { + const firstFake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + firstFake.setIframeFrame(50, "FRAME-SP"); + firstFake.setFrameTree("FRAME-SP", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Frame", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ]); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + for (const [, entry] of state.refs) delete entry.sessionTargetId; + delete state.frameParents; + + const secondFake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + secondFake.setIframeFrame(50, "FRAME-SP"); + secondFake.setFrameTree("FRAME-SP", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Frame", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ]); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + + const resolved = secondFake.sent.find((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 60); + expect(resolved?.sessionId).toBe("session-1"); + }); + it("keeps minting unique refs after import and invalidates imported refs on navigation", async () => { const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); await snapshotText(first); From 79e02212db195c33c5ece3a22fda483435491455 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:16:28 +0000 Subject: [PATCH 03/18] Address remaining review feedback --- packages/agent/src/translator/browser.ts | 10 +++++-- .../agent/test/translator-browser.test.ts | 29 +++++++++++++++++++ packages/ai/src/actions/browser.ts | 3 ++ packages/ai/src/providers/common.ts | 20 ++++++++++--- packages/ai/test/modes.test.ts | 1 + 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 07189ff2..4d3dbac5 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -913,7 +913,12 @@ export class BrowserExecutor { parentSession, ); const frameId = dom.contentDocument?.frameId ?? dom.frameId; - if (!frameId || visitedFrames.has(frameId)) { + if (!frameId) { + complete = false; + continue; + } + if (frameId === targetId) continue; + if (visitedFrames.has(frameId)) { complete = false; continue; } @@ -1380,7 +1385,8 @@ export class BrowserExecutor { this.refs.delete(ref); } } - for (const frameId of invalidatedFrames) this.generations.set(frameId, this.generation(frameId) + 1); + for (const frameId of invalidatedFrames) this.generations.delete(frameId); + this.generations.delete(targetId); } /** SPAs can mint refs indefinitely without ever navigating; bound per-target growth by evicting the oldest. */ diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 03363a8c..f1e9dcd0 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -350,6 +350,17 @@ describe("BrowserExecutor ref lifecycle", () => { await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); + it("drops generation state when a target detaches", async () => { + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + + emit({ method: "Target.detachedFromTarget", params: { sessionId: "session-1", targetId: "TARGET-1" } }); + + expect((executor as unknown as { generations: Map }).generations.has("TARGET-1")).toBe(false); + expect(refsOf(executor).size).toBe(0); + }); + it("does not double-bump the generation for its own navigate", async () => { const { cdp, emit } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); @@ -1375,6 +1386,24 @@ describe("BrowserExecutor browser_act", () => { expect(observedSuccessor(result).diff.url?.after).toBe("https://a.test/saved"); }); + it("does not mark a page-target iframe stub as incomplete", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "TARGET-1"); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Missing", exists: false } }], + }); + + expect(result.stop_reason).toBeUndefined(); + expect(result.steps[0]?.expectation?.status).toBe("preexisting"); + }); + it("does not verify absence against an incomplete nested iframe observation", async () => { const fake = createFakeCdp([ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index c37c80c7..d4af2c1d 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -77,11 +77,13 @@ type CuaBrowserExpectationLeaf = type NonEmptyArray = [T, ...T[]]; +/** Semantic condition evaluated before and after browser action delivery. */ export type CuaBrowserExpectation = | CuaBrowserExpectationLeaf | { all: NonEmptyArray } | { any: NonEmptyArray }; +/** One ref- or focus-based operation in a dependent `browser_act` action list. */ export type CuaBrowserActStep = | { type: "click"; ref: string; button?: "left" | "right" | "middle"; num_clicks?: number; modifiers?: string[]; expect?: CuaBrowserExpectation } | { type: "hover"; ref: string; expect?: CuaBrowserExpectation } @@ -91,6 +93,7 @@ export type CuaBrowserActStep = | { type: "scroll_to"; ref: string; expect?: CuaBrowserExpectation } | { type: "wait"; ms?: number; expect?: CuaBrowserExpectation }; +/** Dependent browser action list with optional per-step and final semantic expectations. */ export interface CuaActionBrowserAct { type: "browser_act"; steps: NonEmptyArray; diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 90463211..2809f318 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -36,11 +36,12 @@ export function buildDefaultComputerSystemPrompt(opts: { suffix?: string } = {}) return [DEFAULT_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } -type ObjectSchemaWithProperties = TSchema & { properties: Record }; +type SchemaWithDefinitions = TSchema & { $defs?: Record }; +type ObjectSchemaWithProperties = SchemaWithDefinitions & { properties: Record }; function createCuaActionArgumentSchema(action: CuaActionType, mode: CuaMode): TSchema { const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); - const schema = schemaByType[action] as ObjectSchemaWithProperties & { $defs?: Record }; + const schema = schemaByType[action] as ObjectSchemaWithProperties; const { type: _type, ...properties } = schema.properties; return Type.Object(properties, { additionalProperties: false, ...(schema.$defs ? { $defs: schema.$defs } : {}) }); } @@ -63,9 +64,20 @@ export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] export const CuaActionSchema = createCuaActionSchema(); export function createCuaBatchSchema(actions?: readonly CuaActionType[], mode: CuaMode = "computer"): TSchema { - return Type.Object({ - actions: Type.Array(createCuaActionSchema(actions, mode), { description: "Ordered computer actions to execute." }), + const actionTypes = actions ?? CUA_ACTION_TYPES; + if (actionTypes.length === 0) throw new Error("actions must include at least one CUA action type"); + const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); + const definitions: Record = {}; + const schemas = actionTypes.map((action) => { + const { $defs, ...schema } = schemaByType[action] as SchemaWithDefinitions; + if ($defs) Object.assign(definitions, $defs); + return schema as TSchema; }); + const actionSchema = schemas.length === 1 ? schemas[0]! : Type.Union(schemas); + return Type.Object( + { actions: Type.Array(actionSchema, { description: "Ordered computer actions to execute." }) }, + Object.keys(definitions).length > 0 ? { $defs: definitions } : {}, + ); } export const CuaBatchSchema = createCuaBatchSchema(); diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index 6f8cb92e..e1703e75 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -103,6 +103,7 @@ describe("mode tool schemas", () => { description: "batch", parameters: createCuaBatchSchema(defaultActionsForMode("browser"), "browser"), }; + expect(batch.parameters.$defs).toBeDefined(); expect(() => validateToolArguments(batch, { id: "call_1", From 366b9a6fcb4aa6df0dc8e9fadc1200e7d8365239 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:25:47 +0000 Subject: [PATCH 04/18] Tighten frame state and schema lifecycles --- packages/agent/src/translator/browser.ts | 48 ++++++++++++++++++- .../agent/test/translator-browser.test.ts | 22 +++++++++ packages/ai/src/providers/common.ts | 27 ++++++----- packages/ai/test/modes.test.ts | 17 ++++++- 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 4d3dbac5..2c7c9c8e 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -377,7 +377,7 @@ export class BrowserExecutor { const targetsBefore = await this.cdp.pageTargets(); const targetBefore = targetsBefore.find((candidate) => candidate.targetId === targetId); if (!targetBefore) throw new ObservationChangedError(); - const generationBefore = this.generation(targetId); + const generationBefore = this.trackGeneration(targetId); const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); const ctx: RenderContext = { targetId, @@ -413,6 +413,7 @@ export class BrowserExecutor { if ([...generations].some(([frameKey, generation]) => this.generation(frameKey) !== generation)) { throw new ObservationChangedError(); } + if (complete) this.pruneFrameState(targetId, new Set(generations.keys())); return { targetId, tree, @@ -924,7 +925,7 @@ export class BrowserExecutor { } visitedFrames.add(frameId); this.frameParents.set(frameId, parentFrameKey); - const generation = this.generation(frameId); + const generation = this.trackGeneration(frameId); const { nodes: childNodes, sessionId } = await this.frameAxTree(frameId, targetId, parentSession); if (generation !== this.generation(frameId)) throw new ObservationChangedError(); const sessionTargetId = this.frameSessions.has(frameId) ? frameId : parentSessionTargetId; @@ -1331,6 +1332,12 @@ export class BrowserExecutor { return this.generations.get(targetId) ?? 0; } + private trackGeneration(targetId: string): number { + const generation = this.generation(targetId); + if (!this.generations.has(targetId)) this.generations.set(targetId, generation); + return generation; + } + private invalidateRefs(targetId: string): void { this.invalidateFrame(targetId); for (const [ref, entry] of this.refs) { @@ -1338,7 +1345,44 @@ export class BrowserExecutor { } } + private pruneFrameState(targetId: string, observed: ReadonlySet): void { + const stale = new Set(); + for (const frameId of this.frameParents.keys()) { + if (observed.has(frameId)) continue; + const visited = new Set([frameId]); + let parent = this.frameParents.get(frameId); + while (parent && !visited.has(parent)) { + if (parent === targetId) { + stale.add(frameId); + break; + } + visited.add(parent); + parent = this.frameParents.get(parent); + } + } + for (const frameId of stale) { + this.frameParents.delete(frameId); + this.frameSessions.delete(frameId); + this.generations.delete(frameId); + } + for (const [ref, entry] of this.refs) { + if (stale.has(entry.frameId) || (entry.sessionTargetId !== undefined && stale.has(entry.sessionTargetId))) { + this.refs.delete(ref); + } + } + } + private invalidateFrame(frameKey: string): void { + let tracked = this.generations.has(frameKey) || this.frameParents.has(frameKey) || this.frameSessions.has(frameKey); + if (!tracked) { + for (const entry of this.refs.values()) { + if (entry.frameId === frameKey || entry.sessionTargetId === frameKey) { + tracked = true; + break; + } + } + } + if (!tracked) return; const invalidated = new Set([frameKey]); let found = true; while (found) { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index f1e9dcd0..5079ee08 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -350,6 +350,28 @@ describe("BrowserExecutor ref lifecycle", () => { await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); + it("prunes generation and ancestry state for rotated iframes", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + const executor = new BrowserExecutor(fake.cdp); + for (let index = 0; index < 5; index += 1) { + const frameId = `FRAME-${index}`; + fake.setIframeFrame(50, frameId); + fake.setFrameTree(frameId, [ax({ nodeId: `f${index}`, role: "RootWebArea", name: `Frame ${index}` })]); + await snapshotText(executor); + if (index < 4) { + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: frameId, parentId: "TARGET-1" } }, sessionId: "session-1" }); + } + } + + const generations = (executor as unknown as { generations: Map }).generations; + const frameParents = (executor as unknown as { frameParents: Map }).frameParents; + expect([...generations.keys()].filter((key) => key.startsWith("FRAME-"))).toEqual(["FRAME-4"]); + expect([...frameParents.keys()]).toEqual(["FRAME-4"]); + }); + it("drops generation state when a target detaches", async () => { const { cdp, emit } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 2809f318..4594b691 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -46,11 +46,22 @@ function createCuaActionArgumentSchema(action: CuaActionType, mode: CuaMode): TS return Type.Object(properties, { additionalProperties: false, ...(schema.$defs ? { $defs: schema.$defs } : {}) }); } -export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): TSchema { +function actionSchemasWithDefinitions(actions: readonly CuaActionType[], mode: CuaMode): { schemas: TSchema[]; definitions: Record } { if (actions.length === 0) throw new Error("actions must include at least one CUA action type"); const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); - if (actions.length === 1) return schemaByType[actions[0]!]; - return Type.Union(actions.map((action) => schemaByType[action])); + const definitions: Record = {}; + const schemas = actions.map((action) => { + const { $defs, ...schema } = schemaByType[action] as SchemaWithDefinitions; + if ($defs) Object.assign(definitions, $defs); + return schema as TSchema; + }); + return { schemas, definitions }; +} + +export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): TSchema { + const { schemas, definitions } = actionSchemasWithDefinitions(actions, mode); + const options = Object.keys(definitions).length > 0 ? { $defs: definitions } : {}; + return schemas.length === 1 ? Object.assign(schemas[0]!, options) : Type.Union(schemas, options); } export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): Tool[] { @@ -64,15 +75,7 @@ export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] export const CuaActionSchema = createCuaActionSchema(); export function createCuaBatchSchema(actions?: readonly CuaActionType[], mode: CuaMode = "computer"): TSchema { - const actionTypes = actions ?? CUA_ACTION_TYPES; - if (actionTypes.length === 0) throw new Error("actions must include at least one CUA action type"); - const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); - const definitions: Record = {}; - const schemas = actionTypes.map((action) => { - const { $defs, ...schema } = schemaByType[action] as SchemaWithDefinitions; - if ($defs) Object.assign(definitions, $defs); - return schema as TSchema; - }); + const { schemas, definitions } = actionSchemasWithDefinitions(actions ?? CUA_ACTION_TYPES, mode); const actionSchema = schemas.length === 1 ? schemas[0]! : Type.Union(schemas); return Type.Object( { actions: Type.Array(actionSchema, { description: "Ordered computer actions to execute." }) }, diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index e1703e75..e406efd1 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -7,6 +7,7 @@ import { CUA_HYBRID_COMPUTER_ACTION_TYPES, anthropic, computerTools, + createCuaActionSchema, createCuaBatchSchema, cuaToolNameForAction, defaultActionsForMode, @@ -97,7 +98,21 @@ describe("mode tool schemas", () => { } }); - it("keeps browser_act definitions resolvable inside a batch schema", () => { + it("keeps browser_act definitions resolvable in action unions and batches", () => { + const union = { + name: "action", + description: "action", + parameters: createCuaActionSchema(defaultActionsForMode("browser"), "browser"), + }; + expect(union.parameters.$defs).toBeDefined(); + expect(() => + validateToolArguments(union, { + id: "call_0", + name: "action", + arguments: { type: "browser_act", steps: [{ type: "wait", ms: 0, expect: { type: "url", changed: true } }] }, + }), + ).not.toThrow(); + const batch = { name: "batch", description: "batch", From 66d24cb58dca98e86dce2656e1ee88fdac7b0376 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:32:41 +0000 Subject: [PATCH 05/18] Prune incomplete frame observations --- packages/agent/src/translator/browser.ts | 2 +- packages/agent/test/translator-browser.test.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 2c7c9c8e..e1a846a7 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -413,7 +413,7 @@ export class BrowserExecutor { if ([...generations].some(([frameKey, generation]) => this.generation(frameKey) !== generation)) { throw new ObservationChangedError(); } - if (complete) this.pruneFrameState(targetId, new Set(generations.keys())); + this.pruneFrameState(targetId, new Set(generations.keys())); return { targetId, tree, diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 5079ee08..fa0ca79e 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -350,10 +350,11 @@ describe("BrowserExecutor ref lifecycle", () => { await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); - it("prunes generation and ancestry state for rotated iframes", async () => { + it("prunes rotated iframe state when another iframe is incomplete", async () => { const fake = createFakeCdp([ - ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 51, parentId: "1" }), ]); const executor = new BrowserExecutor(fake.cdp); for (let index = 0; index < 5; index += 1) { From baaeb88c7babb04e532747fc77446f1fa9739143 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:42:17 +0000 Subject: [PATCH 06/18] Preserve OOPIF refs across nested navigation --- packages/agent/src/tools.ts | 9 ++++++- packages/agent/src/translator/browser.ts | 3 ++- .../agent/test/translator-browser.test.ts | 27 ++++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index f35ff9e2..8acfcc98 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -278,12 +278,19 @@ function formatBrowserActResult(result: BrowserActResult): string { lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.evidence.join("; ")}`); for (const detail of step.expectation?.details ?? []) lines.push(` ${detail}`); } - if (result.final_expectation) lines.push(`final expectation: ${result.final_expectation.status}`); + if (result.final_expectation) { + lines.push(`final expectation: ${result.final_expectation.status}`); + for (const detail of result.final_expectation.details) lines.push(` ${detail}`); + } if (result.successor.status === "unavailable") { lines.push(`successor unavailable: ${result.successor.error}`); } else { const { diff } = result.successor; + lines.push(`successor url: ${result.successor.url}`); + lines.push(`successor title: ${result.successor.title}`); lines.push(`successor diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`); + if (diff.url) lines.push(` url: ${diff.url.before} -> ${diff.url.after}`); + if (diff.title) lines.push(` title: ${diff.title.before} -> ${diff.title.after}`); for (const line of diff.added) lines.push(` + ${line}`); for (const line of diff.removed) lines.push(` - ${line}`); lines.push(result.successor.text); diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index e1a846a7..32a1f393 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -198,7 +198,8 @@ export class BrowserExecutor { const targetId = this.targetsBySession.get(event.sessionId); if (!targetId) return; if (this.frameTargets.has(targetId)) { - this.invalidateSessionTarget(targetId); + if (frame.id && frame.id !== targetId) this.invalidateFrame(frame.id); + else this.invalidateSessionTarget(targetId); return; } if (frame.parentId) { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index fa0ca79e..addec402 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -40,7 +40,20 @@ function createFakeBrowserExecutor() { outcome: stopped ? "didnt" : "unknown", steps: [{ index: 0, type: "click", outcome: stopped ? "didnt" : "unknown", evidence: ["input delivered"] }], ...(stopped ? { stopped_at: 0, stop_reason: "stale_ref" as const } : {}), - successor: { status: "observed", text: 'button "Save" [e2]', url: "https://a.test/", title: "Page", diff: { changed: false, added: [], removed: [] } }, + final_expectation: { status: "preexisting", before: true, after: true, details: ["after: title=\"Page\""] }, + successor: { + status: "observed", + text: 'button "Save" [e2]', + url: "https://a.test/", + title: "Page", + diff: { + changed: true, + added: [], + removed: [], + url: { before: "https://a.test/old", after: "https://a.test/" }, + title: { before: "Old", after: "Page" }, + }, + }, }, }]; } @@ -79,7 +92,12 @@ describe("InternalComputerTranslator browser plane", () => { expect(result.content).toHaveLength(1); expect(result.content[0]).toMatchObject({ type: "text" }); - expect((result.content[0] as { text: string }).text).toContain("browser_act outcome: unknown"); + const text = (result.content[0] as { text: string }).text; + expect(text).toContain("browser_act outcome: unknown"); + expect(text).toContain('after: title="Page"'); + expect(text).toContain("successor url: https://a.test/"); + expect(text).toContain("url: https://a.test/old -> https://a.test/"); + expect(text).toContain("title: Old -> Page"); expect(result.details.statusText).toBe("Browser action outcome is unknown."); expect(result.details.readResults[0]).toMatchObject({ type: "browser_act", result: { outcome: "unknown" } }); }); @@ -1590,6 +1608,7 @@ describe("BrowserExecutor iframe stitching", () => { fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-INNER", parentId: "FRAME-OOP" } }, sessionId: "session-oop" }); await expect(executor.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); }); @@ -1656,13 +1675,13 @@ describe("BrowserExecutor iframe stitching", () => { expect(pressed?.sessionId).toBe("session-1"); }); - it("invalidates a frame target's refs when a subframe inside it navigates", async () => { + it("keeps an OOPIF root ref valid when an unrelated nested frame navigates", async () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); await snapshotText(executor); emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-INNER", parentId: "FRAME-OOP" } }, sessionId: "session-oop" }); - await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).resolves.toEqual([]); }); it("finds elements inside stitched iframes", async () => { From 24a5ffa31798ac77fcefa98e6d727bd34f6bdc9f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:51:41 +0000 Subject: [PATCH 07/18] Bound incomplete frame metadata --- packages/agent/src/translator/browser.ts | 35 ++++++++++++++++++- .../agent/test/translator-browser.test.ts | 26 +++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 32a1f393..7937000a 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -28,6 +28,7 @@ const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; +const FRAME_STATE_LIMIT = 1000; const SCROLL_NOTCH_PX = 120; const EXPECTATION_TIMEOUT_MS = 2_000; const EXPECTATION_POLL_MS = 50; @@ -414,7 +415,9 @@ export class BrowserExecutor { if ([...generations].some(([frameKey, generation]) => this.generation(frameKey) !== generation)) { throw new ObservationChangedError(); } - this.pruneFrameState(targetId, new Set(generations.keys())); + const observedFrames = new Set(generations.keys()); + if (complete) this.pruneFrameState(targetId, observedFrames); + else this.boundFrameState(observedFrames); return { targetId, tree, @@ -1346,6 +1349,36 @@ export class BrowserExecutor { } } + private boundFrameState(observed: ReadonlySet): void { + const protectedFrames = new Set(observed); + for (const frameId of this.frameSessions.keys()) protectedFrames.add(frameId); + for (const entry of this.refs.values()) { + protectedFrames.add(entry.frameId); + if (entry.sessionTargetId) protectedFrames.add(entry.sessionTargetId); + } + let added = true; + while (added) { + added = false; + for (const [child, parent] of this.frameParents) { + if (protectedFrames.has(child) && !protectedFrames.has(parent)) { + protectedFrames.add(parent); + added = true; + } + } + } + for (const frameId of this.frameParents.keys()) { + if (this.frameParents.size <= FRAME_STATE_LIMIT) break; + if (!protectedFrames.has(frameId)) { + this.frameParents.delete(frameId); + this.generations.delete(frameId); + } + } + for (const frameId of this.generations.keys()) { + if (this.generations.size <= FRAME_STATE_LIMIT) break; + if (!protectedFrames.has(frameId)) this.generations.delete(frameId); + } + } + private pruneFrameState(targetId: string, observed: ReadonlySet): void { const stale = new Set(); for (const frameId of this.frameParents.keys()) { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index addec402..bd70c874 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -368,7 +368,7 @@ describe("BrowserExecutor ref lifecycle", () => { await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); - it("prunes rotated iframe state when another iframe is incomplete", async () => { + it("preserves incomplete frame state while bounding stale metadata", async () => { const fake = createFakeCdp([ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), @@ -385,10 +385,26 @@ describe("BrowserExecutor ref lifecycle", () => { } } - const generations = (executor as unknown as { generations: Map }).generations; - const frameParents = (executor as unknown as { frameParents: Map }).frameParents; - expect([...generations.keys()].filter((key) => key.startsWith("FRAME-"))).toEqual(["FRAME-4"]); - expect([...frameParents.keys()]).toEqual(["FRAME-4"]); + const internal = executor as unknown as { + generations: Map; + frameParents: Map; + boundFrameState: (observed: ReadonlySet) => void; + }; + expect([...internal.generations.keys()].filter((key) => key.startsWith("FRAME-"))).toEqual([ + "FRAME-0", + "FRAME-1", + "FRAME-2", + "FRAME-3", + "FRAME-4", + ]); + for (let index = 0; index < 1100; index += 1) { + internal.frameParents.set(`STALE-${index}`, "TARGET-1"); + internal.generations.set(`STALE-${index}`, 0); + } + internal.boundFrameState(new Set(["TARGET-1", "FRAME-4"])); + expect(internal.frameParents.size).toBeLessThanOrEqual(1000); + expect(internal.generations.size).toBeLessThanOrEqual(1000); + expect(internal.frameParents.has("FRAME-4")).toBe(true); }); it("drops generation state when a target detaches", async () => { From fc5a36ab02f3c4c27a0127bf8f81b211ecb6f738 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:03:16 +0000 Subject: [PATCH 08/18] Handle frame edge cases in action results --- packages/agent/src/tools.ts | 9 +- packages/agent/src/translator/browser.ts | 85 ++++++++++++++++++- .../agent/test/translator-browser.test.ts | 83 +++++++++++++++++- 3 files changed, 169 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 8acfcc98..6ad9446d 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -18,6 +18,8 @@ import { InternalComputerTranslator, type KernelBrowser } from "./translator/tra import type { BrowserActResult } from "./translator/types"; import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core"; +const BROWSER_ACT_DIFF_LINE_LIMIT = 200; + export interface ComputerToolOptions { browser: KernelBrowser; client: Kernel; @@ -291,8 +293,11 @@ function formatBrowserActResult(result: BrowserActResult): string { lines.push(`successor diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`); if (diff.url) lines.push(` url: ${diff.url.before} -> ${diff.url.after}`); if (diff.title) lines.push(` title: ${diff.title.before} -> ${diff.title.after}`); - for (const line of diff.added) lines.push(` + ${line}`); - for (const line of diff.removed) lines.push(` - ${line}`); + for (const line of diff.added.slice(0, BROWSER_ACT_DIFF_LINE_LIMIT)) lines.push(` + ${line}`); + const remaining = Math.max(0, BROWSER_ACT_DIFF_LINE_LIMIT - diff.added.length); + for (const line of diff.removed.slice(0, remaining)) lines.push(` - ${line}`); + const omitted = diff.added.length + diff.removed.length - BROWSER_ACT_DIFF_LINE_LIMIT; + if (omitted > 0) lines.push(` … ${omitted} more diff lines omitted`); lines.push(result.successor.text); } return lines.join("\n"); diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 7937000a..2f25ba79 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -94,6 +94,7 @@ interface ObservedNode { interface BrowserObservation { targetId: string; + navigationEpoch: number; tree: FrameStitch; stitches: Map; nodes: ObservedNode[]; @@ -183,6 +184,7 @@ export class BrowserExecutor { private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; private refCounter = 0; + private frameNavigationEpoch = 0; private activeTargetId?: string; private readonly cdp: CdpConnection; @@ -352,7 +354,14 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - const presentation = this.presentObservation(await this.observe(action.tab_id), action); + const observation = await this.observe(action.tab_id); + let presentation: BrowserPresentation; + try { + presentation = this.presentObservation(observation, action); + } catch (err) { + if (!(err instanceof MissingFrameObservationError) || !action.ref) throw err; + presentation = await this.observeReferencedFrame({ ...action, ref: action.ref }, observation.targetId); + } const cached = this.lastSnapshots.get(presentation.observation.targetId); this.lastSnapshots.set(presentation.observation.targetId, presentation); if (cached && cached.key === presentation.key && cached.shape === presentation.shape) return UNCHANGED_SNAPSHOT; @@ -380,6 +389,7 @@ export class BrowserExecutor { const targetBefore = targetsBefore.find((candidate) => candidate.targetId === targetId); if (!targetBefore) throw new ObservationChangedError(); const generationBefore = this.trackGeneration(targetId); + const navigationEpoch = this.frameNavigationEpoch; const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); const ctx: RenderContext = { targetId, @@ -405,6 +415,7 @@ export class BrowserExecutor { if ( !targetAfter || generationBefore !== this.generation(targetId) || + navigationEpoch !== this.frameNavigationEpoch || targetBefore.url !== targetAfter.url || targetBefore.title !== targetAfter.title ) { @@ -420,6 +431,7 @@ export class BrowserExecutor { else this.boundFrameState(observedFrames); return { targetId, + navigationEpoch, tree, stitches, nodes: observedNodes, @@ -430,6 +442,63 @@ export class BrowserExecutor { }; } + private async observeReferencedFrame(action: CuaActionBrowserSnapshot & { ref: string }, targetId: string): Promise { + let changed: ObservationChangedError | undefined; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const entry = this.resolveRef(action.ref, targetId); + const pageSession = await this.attach(targetId); + const targetsBefore = await this.cdp.pageTargets(); + const targetBefore = targetsBefore.find((candidate) => candidate.targetId === targetId); + if (!targetBefore) throw new ObservationChangedError(); + const targetGeneration = this.trackGeneration(targetId); + const frameGeneration = this.trackGeneration(entry.frameId); + const navigationEpoch = this.frameNavigationEpoch; + const { nodes, sessionId } = await this.frameAxTree(entry.frameId, targetId, pageSession); + const targetsAfter = await this.cdp.pageTargets(); + const targetAfter = targetsAfter.find((candidate) => candidate.targetId === targetId); + if ( + !targetAfter || + targetGeneration !== this.generation(targetId) || + frameGeneration !== this.generation(entry.frameId) || + navigationEpoch !== this.frameNavigationEpoch || + targetBefore.url !== targetAfter.url || + targetBefore.title !== targetAfter.title + ) { + throw new ObservationChangedError(); + } + const ctx: RenderContext = { + targetId, + frameKey: entry.frameId, + sessionTargetId: entry.sessionTargetId ?? (this.frameSessions.has(entry.frameId) ? entry.frameId : targetId), + sessionId, + generation: frameGeneration, + nthIndex: buildNthIndex(nodes), + }; + const observation: BrowserObservation = { + targetId, + navigationEpoch, + tree: { + byId: new Map(nodes.map((node) => [node.nodeId, node])), + roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), + ctx, + }, + stitches: new Map(), + nodes: nodes.map((node) => ({ node, ctx })), + url: targetAfter.url, + title: targetAfter.title, + generations: new Map([[targetId, targetGeneration], [entry.frameId, frameGeneration]]), + complete: true, + }; + return this.presentObservation(observation, action); + } catch (err) { + if (!(err instanceof ObservationChangedError)) throw err; + changed = err; + } + } + throw changed ?? new ObservationChangedError(); + } + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { let tree = observation.tree; let rootIds = tree.roots; @@ -439,7 +508,7 @@ export class BrowserExecutor { entry.frameId === observation.tree.ctx.frameKey ? observation.tree : [...observation.stitches.values()].find((candidate) => candidate.ctx.frameKey === entry.frameId); - if (!frameTree) throw staleRefError(action.ref); + if (!frameTree) throw new MissingFrameObservationError(); tree = frameTree; const nodes = [...tree.byId.values()]; const rootNode = nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId) ?? this.healEntry(action.ref, entry, nodes); @@ -621,10 +690,12 @@ export class BrowserExecutor { if (boundaryAfter) { stopReason = boundaryAfter; + } else if (staleRef) { + stopReason = "stale_ref"; } else if (observationError || expectation?.status === "unverifiable") { stopReason = "control_flow"; } else if (actionError) { - stopReason = staleRef ? "stale_ref" : "action_failed"; + stopReason = "action_failed"; } else if (expectation?.status === "failed") { stopReason = "expectation_failed"; } @@ -1417,6 +1488,7 @@ export class BrowserExecutor { } } if (!tracked) return; + this.frameNavigationEpoch += 1; const invalidated = new Set([frameKey]); let found = true; while (found) { @@ -1436,7 +1508,9 @@ export class BrowserExecutor { } } for (const [ref, entry] of this.refs) { - if (invalidated.has(entry.frameId)) this.refs.delete(ref); + if (invalidated.has(entry.frameId) || (entry.sessionTargetId !== undefined && invalidated.has(entry.sessionTargetId))) { + this.refs.delete(ref); + } } } @@ -1524,6 +1598,8 @@ class ObservationChangedError extends Error { } } +class MissingFrameObservationError extends Error {} + interface ExpectationEvaluation { truth?: boolean; details: string[]; @@ -1551,6 +1627,7 @@ function observationGenerationsChanged( after: BrowserObservation, live: ReadonlyMap, ): boolean { + if (before.navigationEpoch !== after.navigationEpoch) return true; for (const [key, generation] of after.generations) { if (generation !== (live.get(key) ?? 0)) return true; const previous = before.generations.get(key); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index bd70c874..c0881ae3 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -34,6 +34,7 @@ function createFakeBrowserExecutor() { if (action.type === "browser_text") return [{ type: "browser_text", label: "text", text: "hello" }]; if (action.type === "browser_act") { const stopped = action.steps[0]?.type === "click" && action.steps[0].ref === "stale"; + const largeDiff = action.steps[0]?.type === "click" && action.steps[0].ref === "large"; return [{ type: "browser_act", result: { @@ -48,7 +49,7 @@ function createFakeBrowserExecutor() { title: "Page", diff: { changed: true, - added: [], + added: largeDiff ? Array.from({ length: 201 }, (_, index) => `line-${index}`) : [], removed: [], url: { before: "https://a.test/old", after: "https://a.test/" }, title: { before: "Old", after: "Page" }, @@ -102,6 +103,20 @@ describe("InternalComputerTranslator browser plane", () => { expect(result.details.readResults[0]).toMatchObject({ type: "browser_act", result: { outcome: "unknown" } }); }); + it("bounds model-facing successor diff output", async () => { + const { client } = createClient(); + const { executor } = createFakeBrowserExecutor(); + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); + const actExecutor = computerToolExecutors({ mode: "browser" }).find((candidate) => candidate.definition.name === "act")!; + const tool = buildCuaComputerTools({ toolExecutors: [actExecutor], mode: "browser" }, translator).find((candidate) => candidate.name === "act")!; + + const result = await tool.execute("call_1", { steps: [{ type: "click", ref: "large" }] }); + const text = (result.content[0] as { text: string }).text; + expect(text).toContain("+ line-199"); + expect(text).not.toContain("+ line-200"); + expect(text).toContain("1 more diff lines omitted"); + }); + it("stops a mixed batch after browser_act reaches a stop boundary", async () => { const { client } = createClient(); const { executor, executed } = createFakeBrowserExecutor(); @@ -990,6 +1005,37 @@ describe("BrowserExecutor browser_act", () => { expect(result.steps).toHaveLength(1); }); + it("stops after a navigated iframe disappears from the successor observation", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "RootWebArea", name: "Frame" })]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + fake.setNodes(BUTTON_TREE); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }, { type: "type", text: "must not run" }], + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "navigation" }); + expect(result.steps).toHaveLength(1); + }); + it("keeps a navigation expectation verified while stopping later dispatch", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; @@ -1528,7 +1574,12 @@ describe("BrowserExecutor browser_act", () => { const executor = new BrowserExecutor(cdp); await snapshotText(executor); emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); - const result = await actResult(executor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "later" }] }); + const result = await actResult(executor, { + steps: [ + { type: "click", ref: "e1", expect: { type: "ref", ref: "e1", checked: true } }, + { type: "type", text: "later" }, + ], + }); expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "stale_ref" }); expect(sent.some((command) => command.method === "Input.insertText")).toBe(false); }); @@ -1555,6 +1606,34 @@ describe("BrowserExecutor iframe stitching", () => { await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); + it("loads a referenced frame directly when page-root stitching is unavailable", async () => { + const fake = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Frame", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ]); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let missStitch = false; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + if (method === "DOM.describeNode" && params?.backendNodeId === 50 && missStitch) { + return { node: { backendNodeId: 50 } }; + } + return inner.send(method, params, sessionId); + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + expect(await snapshotText(executor)).toContain('button "Inside" [e2]'); + missStitch = true; + + expect(await snapshotText(executor, { ref: "e2" })).toContain('button "Inside" [e3]'); + }); + const OOPIF_PAGE = [ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), ax({ nodeId: "2", role: "button", name: "Top", backendDOMNodeId: 40, parentId: "1" }), From dd557067857abf807b05684b96c6eca447e95bb6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:10:06 +0000 Subject: [PATCH 09/18] Avoid duplicate OOPIF invalidation --- packages/agent/src/translator/browser.ts | 2 +- packages/agent/test/translator-browser.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 2f25ba79..f960c31f 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -1517,7 +1517,7 @@ export class BrowserExecutor { private invalidateSessionTarget(targetId: string): void { const ownedFrames = new Set(); for (const entry of this.refs.values()) { - if (entry.sessionTargetId === targetId) ownedFrames.add(entry.frameId); + if (entry.sessionTargetId === targetId && entry.frameId !== targetId) ownedFrames.add(entry.frameId); } this.invalidateFrame(targetId); for (const frameKey of ownedFrames) this.invalidateFrame(frameKey); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index c0881ae3..7f9677a9 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1797,8 +1797,10 @@ describe("BrowserExecutor iframe stitching", () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); await snapshotText(executor); + const generations = (executor as unknown as { generations: Map }).generations; emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-OOP" } }, sessionId: "session-oop" }); + expect(generations.get("FRAME-OOP")).toBe(1); await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); From fb3c6a40bf1741896d8cb890a792249194a6a273 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:19:22 +0000 Subject: [PATCH 10/18] Align semantic checks with rendered snapshots --- packages/agent/src/translator/browser.ts | 27 +++++++-- .../agent/test/translator-browser.test.ts | 56 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index f960c31f..8079b017 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -758,6 +758,7 @@ export class BrowserExecutor { } } + const completed = steps.length === action.steps.length && (!stopReason || terminalNavigation); let successor: BrowserActResult["successor"] | undefined; let successorError: unknown; for (let attempt = 0; attempt < 3 && !successor; attempt += 1) { @@ -804,7 +805,6 @@ export class BrowserExecutor { stoppedAt ??= action.steps.length; } - const completed = steps.length === action.steps.length && (!stopReason || terminalNavigation); const definitiveFailure = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed"; const semanticallyVerified = action.expect ? finalExpectation?.status === "newly_verified" @@ -849,6 +849,23 @@ export class BrowserExecutor { } } + private expectationNodes(observation: BrowserObservation): AXNode[] { + const nodes = observation.nodes.map(({ node }) => node); + for (const tree of [observation.tree, ...observation.stitches.values()]) { + for (const node of tree.byId.values()) { + const childIds = node.childIds ?? []; + for (let index = 0; index < childIds.length; index += 1) { + const run = staticTextRun(tree.byId, childIds, index); + if (run) { + nodes.push(run.node); + index = run.end; + } + } + } + } + return nodes; + } + private evaluateExpectation( expectation: CuaBrowserExpectation, observation: BrowserObservation, @@ -873,16 +890,16 @@ export class BrowserExecutor { return { truth, details: children.flatMap((child) => child.details) }; } if (expectation.type === "text") { - const found = observation.nodes.some( - ({ node }) => !node.ignored && (node.name?.value ?? "").toLowerCase().includes(expectation.text.toLowerCase()), + const found = this.expectationNodes(observation).some( + (node) => !node.ignored && (node.name?.value ?? "").toLowerCase().includes(expectation.text.toLowerCase()), ); const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); const completeness = observation.complete ? "" : "; observation incomplete"; return { truth, details: [`text ${JSON.stringify(expectation.text)} ${found ? "present" : "absent"}${completeness}`] }; } if (expectation.type === "role_name") { - const found = observation.nodes.some( - ({ node }) => + const found = this.expectationNodes(observation).some( + (node) => !node.ignored && (expectation.role === undefined || (node.role?.value ?? "") === expectation.role) && (expectation.name === undefined || (node.name?.value ?? "") === expectation.name), diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 7f9677a9..dd843200 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -955,6 +955,27 @@ describe("BrowserExecutor browser_act", () => { expect(result.steps[0]?.expectation?.status).toBe("preexisting"); }); + it("matches text expectations against merged static-text runs", async () => { + const executor = new BrowserExecutor(createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "StaticText", name: "Hello", parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "world", parentId: "1" }), + ]).cdp); + + const result = await actResult(executor, { + steps: [{ + type: "wait", + ms: 0, + expect: { all: [ + { type: "text", text: "Hello world" }, + { type: "role_name", role: "StaticText", name: "Hello world" }, + ] }, + }], + }); + + expect(result.steps[0]?.expectation?.status).toBe("preexisting"); + }); + it("requires a supplied final expectation to be newly verified", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; @@ -1204,6 +1225,41 @@ describe("BrowserExecutor browser_act", () => { expect(fake.sent.some((command) => command.method === "Input.insertText")).toBe(false); }); + it("keeps a verified outcome when successor collection is unavailable", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + let afterAction = false; + let observationsAfterAction = 0; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + if (method === "Accessibility.getFullAXTree" && afterAction && ++observationsAfterAction > 1) { + throw new Error("successor unavailable"); + } + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "Saved", parentId: "1" }), + ]); + afterAction = true; + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Saved" } }], + }); + + expect(result).toMatchObject({ outcome: "worked", stopped_at: 1, stop_reason: "control_flow" }); + expect(result.steps[0]?.expectation?.status).toBe("newly_verified"); + expect(result.successor.status).toBe("unavailable"); + }); + it("returns an observed successor when dispatch fails after changing the page", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; From 2a507d637b50e01796a7b303773aead65c65f611 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:28:35 +0000 Subject: [PATCH 11/18] Scope navigation epochs by target --- packages/agent/src/translator/browser.ts | 46 +++++++++++++++---- .../agent/test/translator-browser.test.ts | 32 +++++++++++++ 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 8079b017..4134c02a 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -184,7 +184,7 @@ export class BrowserExecutor { private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; private refCounter = 0; - private frameNavigationEpoch = 0; + private readonly navigationEpochs = new Map(); private activeTargetId?: string; private readonly cdp: CdpConnection; @@ -389,7 +389,7 @@ export class BrowserExecutor { const targetBefore = targetsBefore.find((candidate) => candidate.targetId === targetId); if (!targetBefore) throw new ObservationChangedError(); const generationBefore = this.trackGeneration(targetId); - const navigationEpoch = this.frameNavigationEpoch; + const navigationEpoch = this.navigationEpoch(targetId); const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); const ctx: RenderContext = { targetId, @@ -415,7 +415,7 @@ export class BrowserExecutor { if ( !targetAfter || generationBefore !== this.generation(targetId) || - navigationEpoch !== this.frameNavigationEpoch || + navigationEpoch !== this.navigationEpoch(targetId) || targetBefore.url !== targetAfter.url || targetBefore.title !== targetAfter.title ) { @@ -453,7 +453,7 @@ export class BrowserExecutor { if (!targetBefore) throw new ObservationChangedError(); const targetGeneration = this.trackGeneration(targetId); const frameGeneration = this.trackGeneration(entry.frameId); - const navigationEpoch = this.frameNavigationEpoch; + const navigationEpoch = this.navigationEpoch(targetId); const { nodes, sessionId } = await this.frameAxTree(entry.frameId, targetId, pageSession); const targetsAfter = await this.cdp.pageTargets(); const targetAfter = targetsAfter.find((candidate) => candidate.targetId === targetId); @@ -461,7 +461,7 @@ export class BrowserExecutor { !targetAfter || targetGeneration !== this.generation(targetId) || frameGeneration !== this.generation(entry.frameId) || - navigationEpoch !== this.frameNavigationEpoch || + navigationEpoch !== this.navigationEpoch(targetId) || targetBefore.url !== targetAfter.url || targetBefore.title !== targetAfter.title ) { @@ -584,9 +584,21 @@ export class BrowserExecutor { tab_id: action.tab_id, depth: Number.MAX_SAFE_INTEGER, }; - const baseline = await this.observe(action.tab_id); + let baseline: BrowserObservation; + let currentTargets: string[]; + try { + baseline = await this.observe(action.tab_id); + currentTargets = await this.pageTargetIds(); + } catch (err) { + return { + outcome: "unknown", + steps: [], + stopped_at: 0, + stop_reason: "control_flow", + successor: { status: "unavailable", error: errorMessage(err) }, + }; + } let current = baseline; - let currentTargets = await this.pageTargetIds(); let currentDialogCount = this.dialogNotes.length; const steps: BrowserActStepResult[] = []; let stoppedAt: number | undefined; @@ -1430,6 +1442,22 @@ export class BrowserExecutor { return generation; } + private navigationEpoch(targetId: string): number { + return this.navigationEpochs.get(targetId) ?? 0; + } + + private rootFrame(frameKey: string): string { + const visited = new Set(); + let root = frameKey; + while (!visited.has(root)) { + visited.add(root); + const parent = this.frameParents.get(root); + if (!parent) break; + root = parent; + } + return root; + } + private invalidateRefs(targetId: string): void { this.invalidateFrame(targetId); for (const [ref, entry] of this.refs) { @@ -1505,7 +1533,8 @@ export class BrowserExecutor { } } if (!tracked) return; - this.frameNavigationEpoch += 1; + const targetId = this.rootFrame(frameKey); + this.navigationEpochs.set(targetId, this.navigationEpoch(targetId) + 1); const invalidated = new Set([frameKey]); let found = true; while (found) { @@ -1556,6 +1585,7 @@ export class BrowserExecutor { } for (const frameId of invalidatedFrames) this.generations.delete(frameId); this.generations.delete(targetId); + this.navigationEpochs.delete(targetId); } /** SPAs can mint refs indefinitely without ever navigating; bound per-target growth by evicting the oldest. */ diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index dd843200..8733dc18 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -422,6 +422,24 @@ describe("BrowserExecutor ref lifecycle", () => { expect(internal.frameParents.has("FRAME-4")).toBe(true); }); + it("scopes frame navigation epochs to their owning target", async () => { + const executor = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); + await snapshotText(executor); + const internal = executor as unknown as { + generations: Map; + frameParents: Map; + navigationEpochs: Map; + invalidateFrame: (frameId: string) => void; + }; + internal.generations.set("OTHER-FRAME", 0); + internal.frameParents.set("OTHER-FRAME", "OTHER-TARGET"); + + internal.invalidateFrame("OTHER-FRAME"); + + expect(internal.navigationEpochs.get("OTHER-TARGET")).toBe(1); + expect(internal.navigationEpochs.get("TARGET-1")).toBeUndefined(); + }); + it("drops generation state when a target detaches", async () => { const { cdp, emit } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); @@ -877,6 +895,20 @@ describe("BrowserExecutor browser_act", () => { return result.successor; }; + it("returns a structured result when the baseline observation is unavailable", async () => { + const fake = createFakeCdp(BUTTON_TREE); + fake.failOn("Accessibility.getFullAXTree"); + const result = await actResult(new BrowserExecutor(fake.cdp), { steps: [{ type: "wait", ms: 0 }] }); + + expect(result).toMatchObject({ + outcome: "unknown", + steps: [], + stopped_at: 0, + stop_reason: "control_flow", + successor: { status: "unavailable" }, + }); + }); + it("reports a newly verified expectation and a structured successor diff", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; From 806cf286887682756587fb2fb9b49390e1668dee Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:44:06 +0000 Subject: [PATCH 12/18] Stop action batches on iframe detach --- packages/agent/src/translator/browser.ts | 5 +++++ packages/agent/test/translator-browser.test.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 4134c02a..15027613 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -221,6 +221,11 @@ export class BrowserExecutor { if (targetId && frameId === targetId) this.selfNavigations.delete(targetId); return; } + case "Page.frameDetached": { + const { frameId } = event.params as { frameId?: string }; + if (frameId) this.invalidateFrame(frameId); + return; + } case "Target.attachedToTarget": { const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 8733dc18..93b3d833 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1058,7 +1058,7 @@ describe("BrowserExecutor browser_act", () => { expect(result.steps).toHaveLength(1); }); - it("stops after a navigated iframe disappears from the successor observation", async () => { + it("stops after an iframe detaches from the successor observation", async () => { const fake = createFakeCdp([ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), @@ -1072,7 +1072,7 @@ describe("BrowserExecutor browser_act", () => { send: async (method: string, params?: Record, sessionId?: string) => { const result = await inner.send(method, params, sessionId); if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { - fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + fake.emit({ method: "Page.frameDetached", params: { frameId: "FRAME-SP" }, sessionId: "session-1" }); fake.setNodes(BUTTON_TREE); } return result; From 1e61218f934349dedf15808a8b6e4397297098f5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:51:48 +0000 Subject: [PATCH 13/18] Keep attached frame sessions during pruning --- packages/agent/src/translator/browser.ts | 1 - packages/agent/test/translator-browser.test.ts | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 15027613..7703e7d7 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -1517,7 +1517,6 @@ export class BrowserExecutor { } for (const frameId of stale) { this.frameParents.delete(frameId); - this.frameSessions.delete(frameId); this.generations.delete(frameId); } for (const [ref, entry] of this.refs) { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 93b3d833..bf4fbe23 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -403,7 +403,9 @@ describe("BrowserExecutor ref lifecycle", () => { const internal = executor as unknown as { generations: Map; frameParents: Map; + frameSessions: Map; boundFrameState: (observed: ReadonlySet) => void; + pruneFrameState: (targetId: string, observed: ReadonlySet) => void; }; expect([...internal.generations.keys()].filter((key) => key.startsWith("FRAME-"))).toEqual([ "FRAME-0", @@ -420,6 +422,12 @@ describe("BrowserExecutor ref lifecycle", () => { expect(internal.frameParents.size).toBeLessThanOrEqual(1000); expect(internal.generations.size).toBeLessThanOrEqual(1000); expect(internal.frameParents.has("FRAME-4")).toBe(true); + + internal.frameParents.set("ATTACHED-OOPIF", "TARGET-1"); + internal.frameSessions.set("ATTACHED-OOPIF", "session-oop"); + internal.generations.set("ATTACHED-OOPIF", 0); + internal.pruneFrameState("TARGET-1", new Set(["TARGET-1", "FRAME-4"])); + expect(internal.frameSessions.get("ATTACHED-OOPIF")).toBe("session-oop"); }); it("scopes frame navigation epochs to their owning target", async () => { From 14ab4cbcff70984eae937331fdf7cb23b26d973f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:00:08 +0000 Subject: [PATCH 14/18] Stop action batches on SPA navigation --- packages/agent/src/translator/browser.ts | 5 +++- .../agent/test/translator-browser.test.ts | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 7703e7d7..f10b14a2 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -218,7 +218,10 @@ export class BrowserExecutor { if (!event.sessionId) return; const targetId = this.targetsBySession.get(event.sessionId); const { frameId } = event.params as { frameId?: string }; - if (targetId && frameId === targetId) this.selfNavigations.delete(targetId); + if (!targetId || !frameId) return; + if (frameId === targetId && this.selfNavigations.delete(targetId)) return; + const root = this.rootFrame(frameId); + this.navigationEpochs.set(root, this.navigationEpoch(root) + 1); return; } case "Page.frameDetached": { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index bf4fbe23..619ef746 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1097,6 +1097,35 @@ describe("BrowserExecutor browser_act", () => { expect(result.steps).toHaveLength(1); }); + it("stops after a same-document navigation changes page identity", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...fake.cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + const result = await inner.send(method, params, sessionId); + if (method === "Input.dispatchMouseEvent" && params?.type === "mouseReleased") { + fake.setTargets([{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/#next" }]); + fake.emit({ + method: "Page.navigatedWithinDocument", + params: { frameId: "TARGET-1", url: "https://a.test/#next" }, + sessionId: "session-1", + }); + } + return result; + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + + const result = await actResult(executor, { + steps: [{ type: "click", ref: "e1" }, { type: "type", text: "must not run" }], + }); + + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "navigation" }); + expect(result.steps).toHaveLength(1); + }); + it("keeps a navigation expectation verified while stopping later dispatch", async () => { const fake = createFakeCdp(BUTTON_TREE); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; From 56f64bb6ad15587495aa7f7b4a5774b686440a66 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:09:24 +0000 Subject: [PATCH 15/18] Preserve nested frame context in fallback snapshots --- packages/agent/src/translator/browser.ts | 59 +++++++++++++------ .../agent/test/translator-browser.test.ts | 22 ++++++- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index f10b14a2..99ba1cae 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -463,26 +463,40 @@ export class BrowserExecutor { const frameGeneration = this.trackGeneration(entry.frameId); const navigationEpoch = this.navigationEpoch(targetId); const { nodes, sessionId } = await this.frameAxTree(entry.frameId, targetId, pageSession); + const sessionTargetId = entry.sessionTargetId ?? (this.frameSessions.has(entry.frameId) ? entry.frameId : targetId); + const ctx: RenderContext = { + targetId, + frameKey: entry.frameId, + sessionTargetId, + sessionId, + generation: frameGeneration, + nthIndex: buildNthIndex(nodes), + }; + const { frames: stitches, complete } = await this.stitchFrames( + nodes, + targetId, + sessionId, + entry.frameId, + sessionTargetId, + ); const targetsAfter = await this.cdp.pageTargets(); const targetAfter = targetsAfter.find((candidate) => candidate.targetId === targetId); + const generations = new Map([[targetId, targetGeneration], [entry.frameId, frameGeneration]]); + for (const stitch of stitches.values()) generations.set(stitch.ctx.frameKey, stitch.ctx.generation); if ( !targetAfter || - targetGeneration !== this.generation(targetId) || - frameGeneration !== this.generation(entry.frameId) || navigationEpoch !== this.navigationEpoch(targetId) || targetBefore.url !== targetAfter.url || - targetBefore.title !== targetAfter.title + targetBefore.title !== targetAfter.title || + [...generations].some(([frameKey, generation]) => this.generation(frameKey) !== generation) ) { throw new ObservationChangedError(); } - const ctx: RenderContext = { - targetId, - frameKey: entry.frameId, - sessionTargetId: entry.sessionTargetId ?? (this.frameSessions.has(entry.frameId) ? entry.frameId : targetId), - sessionId, - generation: frameGeneration, - nthIndex: buildNthIndex(nodes), - }; + const observedNodes: ObservedNode[] = nodes.map((node) => ({ node, ctx })); + for (const stitch of stitches.values()) { + for (const node of stitch.byId.values()) observedNodes.push({ node, ctx: stitch.ctx }); + } + this.boundFrameState(new Set(generations.keys())); const observation: BrowserObservation = { targetId, navigationEpoch, @@ -491,12 +505,12 @@ export class BrowserExecutor { roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), ctx, }, - stitches: new Map(), - nodes: nodes.map((node) => ({ node, ctx })), + stitches, + nodes: observedNodes, url: targetAfter.url, title: targetAfter.title, - generations: new Map([[targetId, targetGeneration], [entry.frameId, frameGeneration]]), - complete: true, + generations, + complete, }; return this.presentObservation(observation, action); } catch (err) { @@ -1007,7 +1021,13 @@ export class BrowserExecutor { } /** Resolve iframe nodes recursively and fetch each child frame's AX tree for stitching. */ - private async stitchFrames(nodes: AXNode[], targetId: string, pageSession: string): Promise { + private async stitchFrames( + nodes: AXNode[], + targetId: string, + pageSession: string, + rootFrameKey = targetId, + rootSessionTargetId = targetId, + ): Promise { const stitches = new Map(); const visitedFrames = new Set([targetId]); let complete = true; @@ -1061,7 +1081,7 @@ export class BrowserExecutor { } } }; - await visit(nodes, targetId, pageSession, targetId); + await visit(nodes, rootFrameKey, pageSession, rootSessionTargetId); return { frames: stitches, complete }; } @@ -1463,6 +1483,11 @@ export class BrowserExecutor { if (!parent) break; root = parent; } + if (root === frameKey) { + for (const entry of this.refs.values()) { + if (entry.frameId === frameKey || entry.sessionTargetId === frameKey) return entry.targetId; + } + } return root; } diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 619ef746..ecb5cd2f 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1739,7 +1739,13 @@ describe("BrowserExecutor iframe stitching", () => { fake.setIframeFrame(50, "FRAME-SP"); fake.setFrameTree("FRAME-SP", [ ax({ nodeId: "f1", role: "RootWebArea", name: "Frame", childIds: ["f2"] }), - ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ax({ nodeId: "f2", role: "region", name: "Inside", backendDOMNodeId: 62, parentId: "f1", childIds: ["f3"] }), + ax({ nodeId: "f3", role: "Iframe", backendDOMNodeId: 60, parentId: "f2" }), + ]); + fake.setIframeFrame(60, "FRAME-INNER"); + fake.setFrameTree("FRAME-INNER", [ + ax({ nodeId: "g1", role: "RootWebArea", name: "Nested", childIds: ["g2"] }), + ax({ nodeId: "g2", role: "button", name: "Deep", backendDOMNodeId: 70, parentId: "g1" }), ]); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; let missStitch = false; @@ -1753,10 +1759,12 @@ describe("BrowserExecutor iframe stitching", () => { }, } as unknown as CdpConnection; const executor = new BrowserExecutor(wrapped); - expect(await snapshotText(executor)).toContain('button "Inside" [e2]'); + expect(await snapshotText(executor)).toContain('region "Inside" [e2]'); missStitch = true; - expect(await snapshotText(executor, { ref: "e2" })).toContain('button "Inside" [e3]'); + const scoped = await snapshotText(executor, { ref: "e2" }); + expect(scoped).toContain('region "Inside" [e5]'); + expect(scoped).toContain('button "Deep" [e7]'); }); const OOPIF_PAGE = [ @@ -2041,6 +2049,14 @@ describe("BrowserExecutor ref state export/import", () => { const resolved = secondFake.sent.find((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 60); expect(resolved?.sessionId).toBe("session-1"); + + secondFake.emit({ + method: "Page.navigatedWithinDocument", + params: { frameId: "FRAME-SP", url: "https://a.test/#frame" }, + sessionId: "session-1", + }); + const epochs = (second as unknown as { navigationEpochs: Map }).navigationEpochs; + expect(epochs.get("TARGET-1")).toBe(1); }); it("keeps minting unique refs after import and invalidates imported refs on navigation", async () => { From 469ffbf12506f4a224d9db671a9cd370a550095e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:17:27 +0000 Subject: [PATCH 16/18] Guard fallback frame traversal cycles --- packages/agent/src/translator/browser.ts | 2 +- packages/agent/test/translator-browser.test.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 99ba1cae..f082ab62 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -1029,7 +1029,7 @@ export class BrowserExecutor { rootSessionTargetId = targetId, ): Promise { const stitches = new Map(); - const visitedFrames = new Set([targetId]); + const visitedFrames = new Set([targetId, rootFrameKey]); let complete = true; const visit = async ( frameNodes: AXNode[], diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index ecb5cd2f..06149c6c 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -1744,9 +1744,11 @@ describe("BrowserExecutor iframe stitching", () => { ]); fake.setIframeFrame(60, "FRAME-INNER"); fake.setFrameTree("FRAME-INNER", [ - ax({ nodeId: "g1", role: "RootWebArea", name: "Nested", childIds: ["g2"] }), + ax({ nodeId: "g1", role: "RootWebArea", name: "Nested", childIds: ["g2", "g3"] }), ax({ nodeId: "g2", role: "button", name: "Deep", backendDOMNodeId: 70, parentId: "g1" }), + ax({ nodeId: "g3", role: "Iframe", backendDOMNodeId: 71, parentId: "g1" }), ]); + fake.setIframeFrame(71, "FRAME-SP"); const inner = fake.cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; let missStitch = false; const wrapped = { @@ -1763,8 +1765,10 @@ describe("BrowserExecutor iframe stitching", () => { missStitch = true; const scoped = await snapshotText(executor, { ref: "e2" }); - expect(scoped).toContain('region "Inside" [e5]'); - expect(scoped).toContain('button "Deep" [e7]'); + expect(scoped).toContain('region "Inside" [e'); + expect(scoped).toContain('button "Deep" [e'); + const frameParents = (executor as unknown as { frameParents: Map }).frameParents; + expect(frameParents.get("FRAME-SP")).toBe("TARGET-1"); }); const OOPIF_PAGE = [ From 75bd0d2969e93cca08e8f99636e9450c12e76209 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:34:02 +0000 Subject: [PATCH 17/18] Split browser action and observation logic --- packages/agent/src/translator/browser-act.ts | 425 +++++++++++++ .../src/translator/browser-observation.ts | 149 +++++ packages/agent/src/translator/browser.ts | 565 ++---------------- 3 files changed, 609 insertions(+), 530 deletions(-) create mode 100644 packages/agent/src/translator/browser-act.ts create mode 100644 packages/agent/src/translator/browser-observation.ts diff --git a/packages/agent/src/translator/browser-act.ts b/packages/agent/src/translator/browser-act.ts new file mode 100644 index 00000000..48a6f035 --- /dev/null +++ b/packages/agent/src/translator/browser-act.ts @@ -0,0 +1,425 @@ +import type { + CuaActionBrowserAct, + CuaActionBrowserSnapshot, + CuaBrowserActStep, + CuaBrowserExpectation, +} from "@onkernel/cua-ai"; +import { + diffObservations, + ObservationChangedError, + staticTextRun, + type AXNode, + type BrowserObservation, + type BrowserPresentation, +} from "./browser-observation"; +import type { + BrowserActResult, + BrowserActStepResult, + BrowserExpectationEvidence, +} from "./types"; + +const EXPECTATION_TIMEOUT_MS = 2_000; +const EXPECTATION_POLL_MS = 50; + +export interface ExpectationEvaluation { + truth?: boolean; + details: string[]; +} + +type RefExpectation = Extract; + +export interface BrowserActRuntime { + observe(tabId?: string): Promise; + targetIds(): Promise; + dialogCount(): number; + generations(): ReadonlyMap; + executeStep(step: CuaBrowserActStep, tabId?: string): Promise; + evaluateRefExpectation(expectation: RefExpectation, observation: BrowserObservation): ExpectationEvaluation; + presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation; + renderObservation(presentation: BrowserPresentation): string; + rememberPresentation(presentation: BrowserPresentation): void; +} + +export async function runBrowserAct(action: CuaActionBrowserAct, runtime: BrowserActRuntime): Promise { + const observationAction: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor }; + const completeObservationAction: CuaActionBrowserSnapshot = { + type: "browser_snapshot", + tab_id: action.tab_id, + depth: Number.MAX_SAFE_INTEGER, + }; + let baseline: BrowserObservation; + let currentTargets: string[]; + try { + baseline = await runtime.observe(action.tab_id); + currentTargets = await runtime.targetIds(); + } catch (err) { + return { + outcome: "unknown", + steps: [], + stopped_at: 0, + stop_reason: "control_flow", + successor: { status: "unavailable", error: errorMessage(err) }, + }; + } + let current = baseline; + let currentDialogCount = runtime.dialogCount(); + const steps: BrowserActStepResult[] = []; + let stoppedAt: number | undefined; + let stopReason: BrowserActResult["stop_reason"]; + + for (let index = 0; index < action.steps.length; index += 1) { + const step = action.steps[index]!; + const evidence: string[] = []; + let beforeObservation: BrowserObservation; + let beforeTargets: string[]; + try { + beforeObservation = await runtime.observe(action.tab_id); + beforeTargets = await runtime.targetIds(); + } catch (err) { + evidence.push(`pre-action observation failed: ${errorMessage(err)}`); + steps.push({ index, type: step.type, outcome: "unknown", evidence }); + stoppedAt = index; + stopReason = "control_flow"; + break; + } + + const boundary = browserControlChange( + current, + beforeObservation, + runtime.generations(), + currentTargets, + beforeTargets, + currentDialogCount, + runtime.dialogCount(), + ); + current = beforeObservation; + currentTargets = beforeTargets; + currentDialogCount = runtime.dialogCount(); + if (boundary) { + evidence.push(`${boundary} detected before input delivery`); + steps.push({ index, type: step.type, outcome: "unknown", evidence }); + stoppedAt = index; + stopReason = boundary; + break; + } + + const before = step.expect ? evaluateExpectation(step.expect, beforeObservation, baseline, runtime) : undefined; + const dialogCount = currentDialogCount; + let actionError: unknown; + try { + await runtime.executeStep(step, action.tab_id); + evidence.push("input delivered"); + } catch (err) { + actionError = err; + evidence.push(errorMessage(err)); + } + + let expectation: BrowserExpectationEvidence | undefined; + let observationError: unknown; + let boundaryAfter: BrowserActResult["stop_reason"]; + const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; + while (true) { + try { + await delay(0); + const afterObservation = await runtime.observe(action.tab_id); + await delay(0); + const afterTargets = await runtime.targetIds(); + boundaryAfter = browserControlChange( + beforeObservation, + afterObservation, + runtime.generations(), + beforeTargets, + afterTargets, + dialogCount, + runtime.dialogCount(), + ); + current = afterObservation; + currentTargets = afterTargets; + currentDialogCount = runtime.dialogCount(); + if (step.expect) expectation = expectationEvidence(before!, evaluateExpectation(step.expect, current, baseline, runtime)); + if (boundaryAfter && expectation?.status === "failed") { + expectation = { + ...expectation, + status: "unverifiable", + details: [...expectation.details, `${boundaryAfter} interrupted verification`], + }; + } + if (boundaryAfter || !step.expect || expectation?.status !== "failed" || Date.now() >= deadline) break; + await delay(EXPECTATION_POLL_MS); + } catch (err) { + observationError = err; + if (step.expect) { + expectation = { + status: "unverifiable", + before: before?.truth, + details: [...(before?.details ?? []), errorMessage(err)], + }; + } + break; + } + } + + let outcome: BrowserActStepResult["outcome"] = "unknown"; + const staleRef = actionError !== undefined && /ref .* stale/.test(errorMessage(actionError)); + if (expectation?.status === "newly_verified" && actionError === undefined) outcome = "worked"; + else if (expectation?.status === "failed" || staleRef) outcome = "didnt"; + if (expectation) evidence.push(`expectation ${expectation.status}`); + if (observationError) evidence.push(`post-action observation failed: ${errorMessage(observationError)}`); + steps.push({ index, type: step.type, outcome, evidence, ...(expectation ? { expectation } : {}) }); + + if (boundaryAfter) { + stopReason = boundaryAfter; + } else if (staleRef) { + stopReason = "stale_ref"; + } else if (observationError || expectation?.status === "unverifiable") { + stopReason = "control_flow"; + } else if (actionError) { + stopReason = "action_failed"; + } else if (expectation?.status === "failed") { + stopReason = "expectation_failed"; + } + if (stopReason) { + stoppedAt = index; + break; + } + } + + let finalExpectation: BrowserExpectationEvidence | undefined; + const terminalNavigation = + stopReason === "navigation" && + stoppedAt === action.steps.length - 1 && + steps.length === action.steps.length && + steps.at(-1)?.evidence.includes("input delivered") === true; + if (action.expect && (!stopReason || terminalNavigation)) { + const before = evaluateExpectation(action.expect, baseline, baseline, runtime); + const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; + while (true) { + try { + const afterObservation = await runtime.observe(action.tab_id); + await delay(0); + const afterTargets = await runtime.targetIds(); + const boundary = browserControlChange( + current, + afterObservation, + runtime.generations(), + currentTargets, + afterTargets, + currentDialogCount, + runtime.dialogCount(), + ); + current = afterObservation; + currentTargets = afterTargets; + currentDialogCount = runtime.dialogCount(); + if (boundary) { + finalExpectation = { + status: "unverifiable", + before: before.truth, + details: [...(finalExpectation?.details ?? before.details), `${boundary} interrupted verification`], + }; + stopReason = boundary; + break; + } + finalExpectation = expectationEvidence(before, evaluateExpectation(action.expect, current, baseline, runtime)); + if (finalExpectation.status !== "failed" || Date.now() >= deadline) break; + await delay(EXPECTATION_POLL_MS); + } catch (err) { + finalExpectation = { status: "unverifiable", before: before.truth, details: [...before.details, errorMessage(err)] }; + stopReason = "control_flow"; + break; + } + } + if (finalExpectation?.status === "failed") { + stopReason = "expectation_failed"; + stoppedAt = action.steps.length; + } else if (finalExpectation?.status === "unverifiable") { + stopReason ??= "control_flow"; + stoppedAt = action.steps.length; + } + } + + const completed = steps.length === action.steps.length && (!stopReason || terminalNavigation); + let successor: BrowserActResult["successor"] | undefined; + let successorError: unknown; + for (let attempt = 0; attempt < 3 && !successor; attempt += 1) { + try { + const successorObservation = await runtime.observe(action.tab_id); + await delay(0); + const successorTargets = await runtime.targetIds(); + const lateBoundary = browserControlChange( + current, + successorObservation, + runtime.generations(), + currentTargets, + successorTargets, + currentDialogCount, + runtime.dialogCount(), + ); + current = successorObservation; + currentTargets = successorTargets; + currentDialogCount = runtime.dialogCount(); + if (lateBoundary) { + if (!stopReason || stopReason === "control_flow") stopReason = lateBoundary; + stoppedAt ??= action.steps.length; + successorError = new ObservationChangedError(); + continue; + } + const baselineComplete = runtime.presentObservation(baseline, completeObservationAction); + const currentComplete = runtime.presentObservation(current, completeObservationAction); + const currentPresentation = runtime.presentObservation(current, observationAction); + successor = { + status: "observed", + text: runtime.renderObservation(currentPresentation), + url: current.url, + title: current.title, + diff: diffObservations(baselineComplete, currentComplete), + }; + runtime.rememberPresentation(currentPresentation); + } catch (err) { + successorError = err; + } + } + if (!successor) { + successor = { status: "unavailable", error: errorMessage(successorError ?? new ObservationChangedError()) }; + stopReason ??= "control_flow"; + stoppedAt ??= action.steps.length; + } + + const definitiveFailure = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed"; + const semanticallyVerified = action.expect + ? finalExpectation?.status === "newly_verified" + : steps.length > 0 && steps.every((step) => step.outcome === "worked"); + const outcome = definitiveFailure ? "didnt" : completed && semanticallyVerified ? "worked" : "unknown"; + return { + outcome, + steps, + ...(stoppedAt !== undefined ? { stopped_at: stoppedAt } : {}), + ...(stopReason ? { stop_reason: stopReason } : {}), + ...(finalExpectation ? { final_expectation: finalExpectation } : {}), + successor, + }; +} + +function expectationNodes(observation: BrowserObservation): AXNode[] { + const nodes = observation.nodes.map(({ node }) => node); + for (const tree of [observation.tree, ...observation.stitches.values()]) { + for (const node of tree.byId.values()) { + const childIds = node.childIds ?? []; + for (let index = 0; index < childIds.length; index += 1) { + const run = staticTextRun(tree.byId, childIds, index); + if (run) { + nodes.push(run.node); + index = run.end; + } + } + } + } + return nodes; +} + +function evaluateExpectation( + expectation: CuaBrowserExpectation, + observation: BrowserObservation, + baseline: BrowserObservation, + runtime: BrowserActRuntime, +): ExpectationEvaluation { + if ("all" in expectation) { + const children = expectation.all.map((child) => evaluateExpectation(child, observation, baseline, runtime)); + const truth = children.some((child) => child.truth === false) + ? false + : children.some((child) => child.truth === undefined) + ? undefined + : true; + return { truth, details: children.flatMap((child) => child.details) }; + } + if ("any" in expectation) { + const children = expectation.any.map((child) => evaluateExpectation(child, observation, baseline, runtime)); + const truth = children.some((child) => child.truth === true) + ? true + : children.some((child) => child.truth === undefined) + ? undefined + : false; + return { truth, details: children.flatMap((child) => child.details) }; + } + if (expectation.type === "text") { + const found = expectationNodes(observation).some( + (node) => !node.ignored && (node.name?.value ?? "").toLowerCase().includes(expectation.text.toLowerCase()), + ); + const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); + const completeness = observation.complete ? "" : "; observation incomplete"; + return { truth, details: [`text ${JSON.stringify(expectation.text)} ${found ? "present" : "absent"}${completeness}`] }; + } + if (expectation.type === "role_name") { + const found = expectationNodes(observation).some( + (node) => + !node.ignored && + (expectation.role === undefined || (node.role?.value ?? "") === expectation.role) && + (expectation.name === undefined || (node.name?.value ?? "") === expectation.name), + ); + const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); + const completeness = observation.complete ? "" : "; observation incomplete"; + return { truth, details: [`role/name ${found ? "present" : "absent"}${completeness}`] }; + } + if (expectation.type === "url" || expectation.type === "title") { + const value = observation[expectation.type]; + const initial = baseline[expectation.type]; + const checks = [ + expectation.equals === undefined || value === expectation.equals, + expectation.contains === undefined || value.includes(expectation.contains), + expectation.changed === undefined || (value !== initial) === expectation.changed, + ]; + return { truth: checks.every(Boolean), details: [`${expectation.type}=${JSON.stringify(value)}`] }; + } + if (expectation.type !== "ref") return { truth: undefined, details: ["unsupported expectation"] }; + return runtime.evaluateRefExpectation(expectation, observation); +} + +function expectationEvidence(before: ExpectationEvaluation, after: ExpectationEvaluation): BrowserExpectationEvidence { + const details = [...before.details.map((detail) => `before: ${detail}`), ...after.details.map((detail) => `after: ${detail}`)]; + if (before.truth === undefined || after.truth === undefined) { + return { status: "unverifiable", before: before.truth, after: after.truth, details }; + } + if (before.truth && after.truth) return { status: "preexisting", before: true, after: true, details }; + if (!before.truth && after.truth) return { status: "newly_verified", before: false, after: true, details }; + return { status: "failed", before: before.truth, after: after.truth, details }; +} + +function observationGenerationsChanged( + before: BrowserObservation, + after: BrowserObservation, + live: ReadonlyMap, +): boolean { + if (before.navigationEpoch !== after.navigationEpoch) return true; + for (const [key, generation] of after.generations) { + if (generation !== (live.get(key) ?? 0)) return true; + const previous = before.generations.get(key); + if (previous !== undefined && previous !== generation) return true; + } + return false; +} + +function browserControlChange( + before: BrowserObservation, + after: BrowserObservation, + liveGenerations: ReadonlyMap, + beforeTargets: readonly string[], + afterTargets: readonly string[], + beforeDialogCount: number, + afterDialogCount: number, +): BrowserActResult["stop_reason"] { + if (afterDialogCount > beforeDialogCount) return "dialog"; + if (observationGenerationsChanged(before, after, liveGenerations)) return "navigation"; + if (targetsChanged(beforeTargets, afterTargets)) return "control_flow"; + return undefined; +} + +function targetsChanged(before: readonly string[], after: readonly string[]): boolean { + return before.length !== after.length || before.some((targetId, index) => targetId !== after[index]); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts new file mode 100644 index 00000000..ba0629a2 --- /dev/null +++ b/packages/agent/src/translator/browser-observation.ts @@ -0,0 +1,149 @@ +import type { BrowserObservationDiff } from "./types"; + +export const REF_PLACEHOLDER = "\u0000"; + +export interface AXNode { + nodeId: string; + ignored?: boolean; + role?: { value?: string }; + name?: { value?: string }; + value?: { value?: unknown }; + properties?: Array<{ name: string; value?: { value?: unknown } }>; + backendDOMNodeId?: number; + parentId?: string; + childIds?: string[]; +} + +export interface NthIndex { + index: Map; + cohorts: Map; +} + +export interface RenderContext { + targetId: string; + frameKey: string; + sessionTargetId: string; + sessionId: string; + generation: number; + nthIndex: NthIndex; + cursorIds?: ReadonlySet; +} + +export interface ObservationLine { + text: string; + refNode?: AXNode; + ctx: RenderContext; +} + +export interface ObservedNode { + node: AXNode; + ctx: RenderContext; +} + +export interface FrameStitch { + byId: Map; + roots: string[]; + ctx: RenderContext; +} + +export interface FrameStitchResult { + frames: Map; + complete: boolean; +} + +export interface BrowserObservation { + targetId: string; + navigationEpoch: number; + tree: FrameStitch; + stitches: Map; + nodes: ObservedNode[]; + url: string; + title: string; + generations: Map; + complete: boolean; +} + +export interface BrowserPresentation { + observation: BrowserObservation; + key: string; + lines: ObservationLine[]; + shape: string; +} + +export class ObservationChangedError extends Error { + constructor() { + super("page changed while collecting the browser observation"); + } +} + +export class MissingFrameObservationError extends Error {} + +export function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff { + const oldLines = before.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); + const newLines = after.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); + const remaining = new Map(); + for (const line of oldLines) remaining.set(line, (remaining.get(line) ?? 0) + 1); + const added: string[] = []; + for (const line of newLines) { + const count = remaining.get(line) ?? 0; + if (count === 0) added.push(line); + else remaining.set(line, count - 1); + } + const removed: string[] = []; + for (const [line, count] of remaining) { + for (let index = 0; index < count; index += 1) removed.push(line); + } + const url = + before.observation.url === after.observation.url + ? undefined + : { before: before.observation.url, after: after.observation.url }; + const title = + before.observation.title === after.observation.title + ? undefined + : { before: before.observation.title, after: after.observation.title }; + return { + changed: added.length > 0 || removed.length > 0 || url !== undefined || title !== undefined, + added, + removed, + ...(url ? { url } : {}), + ...(title ? { title } : {}), + }; +} + +/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ +export function buildNthIndex(nodes: AXNode[]): NthIndex { + const cohorts = new Map(); + const index = new Map(); + for (const node of nodes) { + if (node.ignored || node.backendDOMNodeId === undefined) continue; + const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); + const nth = cohorts.get(key) ?? 0; + cohorts.set(key, nth + 1); + index.set(node.nodeId, nth); + } + return { index, cohorts }; +} + +export function cohortKey(role: string, name: string): string { + return `${role}\u0000${name}`; +} + +export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { + return `${parentFrameKey}\u0000${backendNodeId}`; +} + +/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ +export function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { + let end = start; + const parts: string[] = []; + while (end < childIds.length) { + const node = tree.get(childIds[end]!); + if (!node || node.ignored || node.role?.value !== "StaticText") break; + const text = node.name?.value ?? ""; + if (text) parts.push(text); + end += 1; + } + if (end - start < 2) return undefined; + const first = tree.get(childIds[start]!)!; + return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; +} diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index f082ab62..219a4965 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -15,14 +15,26 @@ import { type CuaActionBrowserSnapshot, type CuaBrowserAction, } from "@onkernel/cua-ai"; +import { runBrowserAct, type ExpectationEvaluation } from "./browser-act"; +import { + buildNthIndex, + cohortKey, + frameStitchKey, + MissingFrameObservationError, + ObservationChangedError, + REF_PLACEHOLDER, + staticTextRun, + type AXNode, + type BrowserObservation, + type BrowserPresentation, + type FrameStitch, + type FrameStitchResult, + type ObservationLine, + type ObservedNode, + type RenderContext, +} from "./browser-observation"; import { CdpConnection, type CdpEventMessage } from "./cdp"; -import type { - BatchReadResult, - BrowserActResult, - BrowserActStepResult, - BrowserExpectationEvidence, - BrowserObservationDiff, -} from "./types"; +import type { BatchReadResult, BrowserActResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; @@ -30,25 +42,10 @@ const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; const FRAME_STATE_LIMIT = 1000; const SCROLL_NOTCH_PX = 120; -const EXPECTATION_TIMEOUT_MS = 2_000; -const EXPECTATION_POLL_MS = 50; const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; -const REF_PLACEHOLDER = "\u0000"; const UNCHANGED_SNAPSHOT = "Page unchanged since the last snapshot; previous element refs are still valid."; -interface AXNode { - nodeId: string; - ignored?: boolean; - role?: { value?: string }; - name?: { value?: string }; - value?: { value?: unknown }; - properties?: Array<{ name: string; value?: { value?: unknown } }>; - backendDOMNodeId?: number; - parentId?: string; - childIds?: string[]; -} - interface RefEntry { backendNodeId: number; targetId: string; @@ -66,62 +63,6 @@ interface RefEntry { cohort: number; } -interface NthIndex { - index: Map; - cohorts: Map; -} - -interface RenderContext { - targetId: string; - frameKey: string; - sessionTargetId: string; - sessionId: string; - generation: number; - nthIndex: NthIndex; - cursorIds?: ReadonlySet; -} - -interface ObservationLine { - text: string; - refNode?: AXNode; - ctx: RenderContext; -} - -interface ObservedNode { - node: AXNode; - ctx: RenderContext; -} - -interface BrowserObservation { - targetId: string; - navigationEpoch: number; - tree: FrameStitch; - stitches: Map; - nodes: ObservedNode[]; - url: string; - title: string; - generations: Map; - complete: boolean; -} - -interface BrowserPresentation { - observation: BrowserObservation; - key: string; - lines: ObservationLine[]; - shape: string; -} - -interface FrameStitch { - byId: Map; - roots: string[]; - ctx: RenderContext; -} - -interface FrameStitchResult { - frames: Map; - complete: boolean; -} - export interface BrowserFindCandidate { ref: string; role: string; @@ -599,259 +540,20 @@ export class BrowserExecutor { return text || "(empty accessibility tree)"; } - private async act(action: CuaActionBrowserAct): Promise { - const observationAction: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor }; - const completeObservationAction: CuaActionBrowserSnapshot = { - type: "browser_snapshot", - tab_id: action.tab_id, - depth: Number.MAX_SAFE_INTEGER, - }; - let baseline: BrowserObservation; - let currentTargets: string[]; - try { - baseline = await this.observe(action.tab_id); - currentTargets = await this.pageTargetIds(); - } catch (err) { - return { - outcome: "unknown", - steps: [], - stopped_at: 0, - stop_reason: "control_flow", - successor: { status: "unavailable", error: errorMessage(err) }, - }; - } - let current = baseline; - let currentDialogCount = this.dialogNotes.length; - const steps: BrowserActStepResult[] = []; - let stoppedAt: number | undefined; - let stopReason: BrowserActResult["stop_reason"]; - - for (let index = 0; index < action.steps.length; index += 1) { - const step = action.steps[index]!; - const evidence: string[] = []; - let beforeObservation: BrowserObservation; - let beforeTargets: string[]; - try { - beforeObservation = await this.observe(action.tab_id); - beforeTargets = await this.pageTargetIds(); - } catch (err) { - evidence.push(`pre-action observation failed: ${errorMessage(err)}`); - steps.push({ index, type: step.type, outcome: "unknown", evidence }); - stoppedAt = index; - stopReason = "control_flow"; - break; - } - - const boundary = browserControlChange( - current, - beforeObservation, - this.generations, - currentTargets, - beforeTargets, - currentDialogCount, - this.dialogNotes.length, - ); - current = beforeObservation; - currentTargets = beforeTargets; - currentDialogCount = this.dialogNotes.length; - if (boundary) { - evidence.push(`${boundary} detected before input delivery`); - steps.push({ index, type: step.type, outcome: "unknown", evidence }); - stoppedAt = index; - stopReason = boundary; - break; - } - - const before = step.expect ? this.evaluateExpectation(step.expect, beforeObservation, baseline) : undefined; - const dialogCount = currentDialogCount; - let actionError: unknown; - try { - await this.executeActStep(step, action.tab_id); - evidence.push("input delivered"); - } catch (err) { - actionError = err; - evidence.push(errorMessage(err)); - } - - let expectation: BrowserExpectationEvidence | undefined; - let observationError: unknown; - let boundaryAfter: BrowserActResult["stop_reason"]; - const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; - while (true) { - try { - await delay(0); - const afterObservation = await this.observe(action.tab_id); - await delay(0); - const afterTargets = await this.pageTargetIds(); - boundaryAfter = browserControlChange( - beforeObservation, - afterObservation, - this.generations, - beforeTargets, - afterTargets, - dialogCount, - this.dialogNotes.length, - ); - current = afterObservation; - currentTargets = afterTargets; - currentDialogCount = this.dialogNotes.length; - if (step.expect) expectation = expectationEvidence(before!, this.evaluateExpectation(step.expect, current, baseline)); - if (boundaryAfter && expectation?.status === "failed") { - expectation = { ...expectation, status: "unverifiable", details: [...expectation.details, `${boundaryAfter} interrupted verification`] }; - } - if (boundaryAfter || !step.expect || expectation?.status !== "failed" || Date.now() >= deadline) break; - await delay(EXPECTATION_POLL_MS); - } catch (err) { - observationError = err; - if (step.expect) { - expectation = { - status: "unverifiable", - before: before?.truth, - details: [...(before?.details ?? []), errorMessage(err)], - }; - } - break; - } - } - - let outcome: BrowserActStepResult["outcome"] = "unknown"; - const staleRef = actionError !== undefined && /ref .* stale/.test(errorMessage(actionError)); - if (expectation?.status === "newly_verified" && actionError === undefined) outcome = "worked"; - else if (expectation?.status === "failed" || staleRef) outcome = "didnt"; - if (expectation) evidence.push(`expectation ${expectation.status}`); - if (observationError) evidence.push(`post-action observation failed: ${errorMessage(observationError)}`); - steps.push({ index, type: step.type, outcome, evidence, ...(expectation ? { expectation } : {}) }); - - if (boundaryAfter) { - stopReason = boundaryAfter; - } else if (staleRef) { - stopReason = "stale_ref"; - } else if (observationError || expectation?.status === "unverifiable") { - stopReason = "control_flow"; - } else if (actionError) { - stopReason = "action_failed"; - } else if (expectation?.status === "failed") { - stopReason = "expectation_failed"; - } - if (stopReason) { - stoppedAt = index; - break; - } - } - - let finalExpectation: BrowserExpectationEvidence | undefined; - const terminalNavigation = - stopReason === "navigation" && - stoppedAt === action.steps.length - 1 && - steps.length === action.steps.length && - steps.at(-1)?.evidence.includes("input delivered") === true; - if (action.expect && (!stopReason || terminalNavigation)) { - const before = this.evaluateExpectation(action.expect, baseline, baseline); - const deadline = Date.now() + EXPECTATION_TIMEOUT_MS; - while (true) { - try { - const afterObservation = await this.observe(action.tab_id); - await delay(0); - const afterTargets = await this.pageTargetIds(); - const boundary = browserControlChange( - current, - afterObservation, - this.generations, - currentTargets, - afterTargets, - currentDialogCount, - this.dialogNotes.length, - ); - current = afterObservation; - currentTargets = afterTargets; - currentDialogCount = this.dialogNotes.length; - if (boundary) { - finalExpectation = { - status: "unverifiable", - before: before.truth, - details: [...(finalExpectation?.details ?? before.details), `${boundary} interrupted verification`], - }; - stopReason = boundary; - break; - } - finalExpectation = expectationEvidence(before, this.evaluateExpectation(action.expect, current, baseline)); - if (finalExpectation.status !== "failed" || Date.now() >= deadline) break; - await delay(EXPECTATION_POLL_MS); - } catch (err) { - finalExpectation = { status: "unverifiable", before: before.truth, details: [...before.details, errorMessage(err)] }; - stopReason = "control_flow"; - break; - } - } - if (finalExpectation?.status === "failed") { - stopReason = "expectation_failed"; - stoppedAt = action.steps.length; - } else if (finalExpectation?.status === "unverifiable") { - stopReason ??= "control_flow"; - stoppedAt = action.steps.length; - } - } - - const completed = steps.length === action.steps.length && (!stopReason || terminalNavigation); - let successor: BrowserActResult["successor"] | undefined; - let successorError: unknown; - for (let attempt = 0; attempt < 3 && !successor; attempt += 1) { - try { - const successorObservation = await this.observe(action.tab_id); - await delay(0); - const successorTargets = await this.pageTargetIds(); - const lateBoundary = browserControlChange( - current, - successorObservation, - this.generations, - currentTargets, - successorTargets, - currentDialogCount, - this.dialogNotes.length, - ); - current = successorObservation; - currentTargets = successorTargets; - currentDialogCount = this.dialogNotes.length; - if (lateBoundary) { - if (!stopReason || stopReason === "control_flow") stopReason = lateBoundary; - stoppedAt ??= action.steps.length; - successorError = new ObservationChangedError(); - continue; - } - const baselineComplete = this.presentObservation(baseline, completeObservationAction); - const currentComplete = this.presentObservation(current, completeObservationAction); - const currentPresentation = this.presentObservation(current, observationAction); - successor = { - status: "observed", - text: this.renderObservation(currentPresentation), - url: current.url, - title: current.title, - diff: diffObservations(baselineComplete, currentComplete), - }; - this.lastSnapshots.set(current.targetId, currentPresentation); - } catch (err) { - successorError = err; - } - } - if (!successor) { - successor = { status: "unavailable", error: errorMessage(successorError ?? new ObservationChangedError()) }; - stopReason ??= "control_flow"; - stoppedAt ??= action.steps.length; - } - - const definitiveFailure = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed"; - const semanticallyVerified = action.expect - ? finalExpectation?.status === "newly_verified" - : steps.length > 0 && steps.every((step) => step.outcome === "worked"); - const outcome = definitiveFailure ? "didnt" : completed && semanticallyVerified ? "worked" : "unknown"; - return { - outcome, - steps, - ...(stoppedAt !== undefined ? { stopped_at: stoppedAt } : {}), - ...(stopReason ? { stop_reason: stopReason } : {}), - ...(finalExpectation ? { final_expectation: finalExpectation } : {}), - successor, - }; + private act(action: CuaActionBrowserAct): Promise { + return runBrowserAct(action, { + observe: (tabId) => this.observe(tabId), + targetIds: () => this.pageTargetIds(), + dialogCount: () => this.dialogNotes.length, + generations: () => this.generations, + executeStep: (step, tabId) => this.executeActStep(step, tabId), + evaluateRefExpectation: (expectation, observation) => this.evaluateRefExpectation(expectation, observation), + presentObservation: (observation, snapshot) => this.presentObservation(observation, snapshot), + renderObservation: (presentation) => this.renderObservation(presentation), + rememberPresentation: (presentation) => { + this.lastSnapshots.set(presentation.observation.targetId, presentation); + }, + }); } private async executeActStep(step: CuaBrowserActStep, tabId?: string): Promise { @@ -883,77 +585,10 @@ export class BrowserExecutor { } } - private expectationNodes(observation: BrowserObservation): AXNode[] { - const nodes = observation.nodes.map(({ node }) => node); - for (const tree of [observation.tree, ...observation.stitches.values()]) { - for (const node of tree.byId.values()) { - const childIds = node.childIds ?? []; - for (let index = 0; index < childIds.length; index += 1) { - const run = staticTextRun(tree.byId, childIds, index); - if (run) { - nodes.push(run.node); - index = run.end; - } - } - } - } - return nodes; - } - - private evaluateExpectation( - expectation: CuaBrowserExpectation, + private evaluateRefExpectation( + expectation: Extract, observation: BrowserObservation, - baseline: BrowserObservation, ): ExpectationEvaluation { - if ("all" in expectation) { - const children = expectation.all.map((child) => this.evaluateExpectation(child, observation, baseline)); - const truth = children.some((child) => child.truth === false) - ? false - : children.some((child) => child.truth === undefined) - ? undefined - : true; - return { truth, details: children.flatMap((child) => child.details) }; - } - if ("any" in expectation) { - const children = expectation.any.map((child) => this.evaluateExpectation(child, observation, baseline)); - const truth = children.some((child) => child.truth === true) - ? true - : children.some((child) => child.truth === undefined) - ? undefined - : false; - return { truth, details: children.flatMap((child) => child.details) }; - } - if (expectation.type === "text") { - const found = this.expectationNodes(observation).some( - (node) => !node.ignored && (node.name?.value ?? "").toLowerCase().includes(expectation.text.toLowerCase()), - ); - const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); - const completeness = observation.complete ? "" : "; observation incomplete"; - return { truth, details: [`text ${JSON.stringify(expectation.text)} ${found ? "present" : "absent"}${completeness}`] }; - } - if (expectation.type === "role_name") { - const found = this.expectationNodes(observation).some( - (node) => - !node.ignored && - (expectation.role === undefined || (node.role?.value ?? "") === expectation.role) && - (expectation.name === undefined || (node.name?.value ?? "") === expectation.name), - ); - const truth = !found && !observation.complete ? undefined : found === (expectation.exists ?? true); - const completeness = observation.complete ? "" : "; observation incomplete"; - return { truth, details: [`role/name ${found ? "present" : "absent"}${completeness}`] }; - } - if (expectation.type === "url" || expectation.type === "title") { - const value = observation[expectation.type]; - const initial = baseline[expectation.type]; - const checks = [ - expectation.equals === undefined || value === expectation.equals, - expectation.contains === undefined || value.includes(expectation.contains), - expectation.changed === undefined || (value !== initial) === expectation.changed, - ]; - return { truth: checks.every(Boolean), details: [`${expectation.type}=${JSON.stringify(value)}`] }; - } - if (expectation.type !== "ref") return { truth: undefined, details: ["unsupported expectation"] }; - const entry = this.refs.get(expectation.ref); if (!entry || entry.targetId !== observation.targetId || entry.generation !== this.generation(entry.frameId)) { return { truth: undefined, details: [`ref ${expectation.ref} is stale`] }; @@ -1671,29 +1306,6 @@ export class BrowserExecutor { } } -class ObservationChangedError extends Error { - constructor() { - super("page changed while collecting the browser observation"); - } -} - -class MissingFrameObservationError extends Error {} - -interface ExpectationEvaluation { - truth?: boolean; - details: string[]; -} - -function expectationEvidence(before: ExpectationEvaluation, after: ExpectationEvaluation): BrowserExpectationEvidence { - const details = [...before.details.map((detail) => `before: ${detail}`), ...after.details.map((detail) => `after: ${detail}`)]; - if (before.truth === undefined || after.truth === undefined) { - return { status: "unverifiable", before: before.truth, after: after.truth, details }; - } - if (before.truth && after.truth) return { status: "preexisting", before: true, after: true, details }; - if (!before.truth && after.truth) return { status: "newly_verified", before: false, after: true, details }; - return { status: "failed", before: before.truth, after: after.truth, details }; -} - function normalizeState(value: unknown): boolean | "mixed" | undefined { if (value === true || value === "true") return true; if (value === false || value === "false") return false; @@ -1701,75 +1313,6 @@ function normalizeState(value: unknown): boolean | "mixed" | undefined { return undefined; } -function observationGenerationsChanged( - before: BrowserObservation, - after: BrowserObservation, - live: ReadonlyMap, -): boolean { - if (before.navigationEpoch !== after.navigationEpoch) return true; - for (const [key, generation] of after.generations) { - if (generation !== (live.get(key) ?? 0)) return true; - const previous = before.generations.get(key); - if (previous !== undefined && previous !== generation) return true; - } - return false; -} - -function browserControlChange( - before: BrowserObservation, - after: BrowserObservation, - liveGenerations: ReadonlyMap, - beforeTargets: readonly string[], - afterTargets: readonly string[], - beforeDialogCount: number, - afterDialogCount: number, -): BrowserActResult["stop_reason"] { - if (afterDialogCount > beforeDialogCount) return "dialog"; - if (observationGenerationsChanged(before, after, liveGenerations)) return "navigation"; - if (targetsChanged(beforeTargets, afterTargets)) return "control_flow"; - return undefined; -} - -function targetsChanged(before: readonly string[], after: readonly string[]): boolean { - return before.length !== after.length || before.some((targetId, index) => targetId !== after[index]); -} - -function delay(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff { - const oldLines = before.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); - const newLines = after.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); - const remaining = new Map(); - for (const line of oldLines) remaining.set(line, (remaining.get(line) ?? 0) + 1); - const added: string[] = []; - for (const line of newLines) { - const count = remaining.get(line) ?? 0; - if (count === 0) added.push(line); - else remaining.set(line, count - 1); - } - const removed: string[] = []; - for (const [line, count] of remaining) { - for (let index = 0; index < count; index += 1) removed.push(line); - } - const url = - before.observation.url === after.observation.url - ? undefined - : { before: before.observation.url, after: after.observation.url }; - const title = - before.observation.title === after.observation.title - ? undefined - : { before: before.observation.title, after: after.observation.title }; - return { - changed: added.length > 0 || removed.length > 0 || url !== undefined || title !== undefined, - added, - removed, - ...(url ? { url } : {}), - ...(title ? { title } : {}), - }; -} - function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -1782,44 +1325,6 @@ function staleRefError(ref: string, cause?: unknown): Error { return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); } -/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ -function buildNthIndex(nodes: AXNode[]): NthIndex { - const cohorts = new Map(); - const index = new Map(); - for (const node of nodes) { - if (node.ignored || node.backendDOMNodeId === undefined) continue; - const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); - const nth = cohorts.get(key) ?? 0; - cohorts.set(key, nth + 1); - index.set(node.nodeId, nth); - } - return { index, cohorts }; -} - -function cohortKey(role: string, name: string): string { - return `${role}\u0000${name}`; -} - -function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { - return `${parentFrameKey}\u0000${backendNodeId}`; -} - -/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ -function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { - let end = start; - const parts: string[] = []; - while (end < childIds.length) { - const node = tree.get(childIds[end]!); - if (!node || node.ignored || node.role?.value !== "StaticText") break; - const text = node.name?.value ?? ""; - if (text) parts.push(text); - end += 1; - } - if (end - start < 2) return undefined; - const first = tree.get(childIds[start]!)!; - return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; -} - function collectStates(node: AXNode): string[] { const states: string[] = []; for (const property of node.properties ?? []) { From 0e9edb8e7f6e0b1e9cc33d08d03219f9ad37418d Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:42:06 +0000 Subject: [PATCH 18/18] Document modular browser internals --- packages/agent/src/translator/browser-act.ts | 14 ++++++++++++-- .../agent/src/translator/browser-observation.ts | 15 +++++++++++++++ packages/agent/src/translator/browser.ts | 5 +---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/translator/browser-act.ts b/packages/agent/src/translator/browser-act.ts index 48a6f035..e96b0fb4 100644 --- a/packages/agent/src/translator/browser-act.ts +++ b/packages/agent/src/translator/browser-act.ts @@ -21,6 +21,7 @@ import type { const EXPECTATION_TIMEOUT_MS = 2_000; const EXPECTATION_POLL_MS = 50; +/** Internal three-valued result produced while checking a browser expectation. */ export interface ExpectationEvaluation { truth?: boolean; details: string[]; @@ -28,11 +29,13 @@ export interface ExpectationEvaluation { type RefExpectation = Extract; +/** Browser capabilities required by the dependent action state machine. */ export interface BrowserActRuntime { observe(tabId?: string): Promise; targetIds(): Promise; dialogCount(): number; generations(): ReadonlyMap; + navigationEpoch(targetId: string): number; executeStep(step: CuaBrowserActStep, tabId?: string): Promise; evaluateRefExpectation(expectation: RefExpectation, observation: BrowserObservation): ExpectationEvaluation; presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation; @@ -40,6 +43,7 @@ export interface BrowserActRuntime { rememberPresentation(presentation: BrowserPresentation): void; } +/** Execute a dependent browser action list against an injected browser runtime. */ export async function runBrowserAct(action: CuaActionBrowserAct, runtime: BrowserActRuntime): Promise { const observationAction: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor }; const completeObservationAction: CuaActionBrowserSnapshot = { @@ -87,6 +91,7 @@ export async function runBrowserAct(action: CuaActionBrowserAct, runtime: Browse current, beforeObservation, runtime.generations(), + runtime.navigationEpoch(beforeObservation.targetId), currentTargets, beforeTargets, currentDialogCount, @@ -128,6 +133,7 @@ export async function runBrowserAct(action: CuaActionBrowserAct, runtime: Browse beforeObservation, afterObservation, runtime.generations(), + runtime.navigationEpoch(afterObservation.targetId), beforeTargets, afterTargets, dialogCount, @@ -202,6 +208,7 @@ export async function runBrowserAct(action: CuaActionBrowserAct, runtime: Browse current, afterObservation, runtime.generations(), + runtime.navigationEpoch(afterObservation.targetId), currentTargets, afterTargets, currentDialogCount, @@ -249,6 +256,7 @@ export async function runBrowserAct(action: CuaActionBrowserAct, runtime: Browse current, successorObservation, runtime.generations(), + runtime.navigationEpoch(successorObservation.targetId), currentTargets, successorTargets, currentDialogCount, @@ -387,8 +395,9 @@ function observationGenerationsChanged( before: BrowserObservation, after: BrowserObservation, live: ReadonlyMap, + liveNavigationEpoch: number, ): boolean { - if (before.navigationEpoch !== after.navigationEpoch) return true; + if (before.navigationEpoch !== after.navigationEpoch || after.navigationEpoch !== liveNavigationEpoch) return true; for (const [key, generation] of after.generations) { if (generation !== (live.get(key) ?? 0)) return true; const previous = before.generations.get(key); @@ -401,13 +410,14 @@ function browserControlChange( before: BrowserObservation, after: BrowserObservation, liveGenerations: ReadonlyMap, + liveNavigationEpoch: number, beforeTargets: readonly string[], afterTargets: readonly string[], beforeDialogCount: number, afterDialogCount: number, ): BrowserActResult["stop_reason"] { if (afterDialogCount > beforeDialogCount) return "dialog"; - if (observationGenerationsChanged(before, after, liveGenerations)) return "navigation"; + if (observationGenerationsChanged(before, after, liveGenerations, liveNavigationEpoch)) return "navigation"; if (targetsChanged(beforeTargets, afterTargets)) return "control_flow"; return undefined; } diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts index ba0629a2..de9f7346 100644 --- a/packages/agent/src/translator/browser-observation.ts +++ b/packages/agent/src/translator/browser-observation.ts @@ -1,7 +1,9 @@ import type { BrowserObservationDiff } from "./types"; +/** Internal marker replaced with a minted element ref during rendering. */ export const REF_PLACEHOLDER = "\u0000"; +/** Minimal CDP accessibility node shape used by browser observation code. */ export interface AXNode { nodeId: string; ignored?: boolean; @@ -14,11 +16,13 @@ export interface AXNode { childIds?: string[]; } +/** Role/name cohort positions used for conservative ref healing. */ export interface NthIndex { index: Map; cohorts: Map; } +/** Frame and generation metadata needed to render and mint refs. */ export interface RenderContext { targetId: string; frameKey: string; @@ -29,28 +33,33 @@ export interface RenderContext { cursorIds?: ReadonlySet; } +/** One normalized accessibility line before ref placeholders are resolved. */ export interface ObservationLine { text: string; refNode?: AXNode; ctx: RenderContext; } +/** Accessibility node paired with its owning frame context. */ export interface ObservedNode { node: AXNode; ctx: RenderContext; } +/** Accessibility tree collected for one stitched child frame. */ export interface FrameStitch { byId: Map; roots: string[]; ctx: RenderContext; } +/** Collected child frames plus whether the traversal was complete. */ export interface FrameStitchResult { frames: Map; complete: boolean; } +/** Complete structured browser state collected before presentation filtering. */ export interface BrowserObservation { targetId: string; navigationEpoch: number; @@ -63,6 +72,7 @@ export interface BrowserObservation { complete: boolean; } +/** Render-ready projection of a structured browser observation. */ export interface BrowserPresentation { observation: BrowserObservation; key: string; @@ -70,14 +80,17 @@ export interface BrowserPresentation { shape: string; } +/** Signals that browser state changed while an observation was being collected. */ export class ObservationChangedError extends Error { constructor() { super("page changed while collecting the browser observation"); } } +/** Signals that a referenced frame was absent from the page-root stitched observation. */ export class MissingFrameObservationError extends Error {} +/** Compute a complete normalized successor diff without exposing unstable refs. */ export function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff { const oldLines = before.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); const newLines = after.lines.map((line) => line.text.replace(REF_PLACEHOLDER, "ref")); @@ -124,10 +137,12 @@ export function buildNthIndex(nodes: AXNode[]): NthIndex { return { index, cohorts }; } +/** Build the stable key for a role/name healing cohort. */ export function cohortKey(role: string, name: string): string { return `${role}\u0000${name}`; } +/** Build the lookup key for an iframe node's stitched child tree. */ export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { return `${parentFrameKey}\u0000${backendNodeId}`; } diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 219a4965..1c9ef9ec 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -546,6 +546,7 @@ export class BrowserExecutor { targetIds: () => this.pageTargetIds(), dialogCount: () => this.dialogNotes.length, generations: () => this.generations, + navigationEpoch: (targetId) => this.navigationEpoch(targetId), executeStep: (step, tabId) => this.executeActStep(step, tabId), evaluateRefExpectation: (expectation, observation) => this.evaluateRefExpectation(expectation, observation), presentObservation: (observation, snapshot) => this.presentObservation(observation, snapshot), @@ -1313,10 +1314,6 @@ function normalizeState(value: unknown): boolean | "mixed" | undefined { return undefined; } -function errorMessage(err: unknown): string { - return err instanceof Error ? err.message : String(err); -} - function tabOf(action: { tab_id?: string }): string | undefined { return action.tab_id; }