Add verified browser action batches

packages/agent/src/index.ts

@@ -6,7 +6,19 @@ export { InternalComputerTranslator } from "./translator/translator";
 export { CdpConnection } from "./translator/cdp";
 export { BrowserExecutor } from "./translator/browser";
 export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser";
-export type { BatchExecutionResult, BatchReadResult } from "./translator/types";
+export type {
+	BatchExecutionResult,
+	BatchReadResult,
+	BrowserActObservedSuccessor,
+	BrowserActOutcome,
+	BrowserActResult,
+	BrowserActStepResult,
+	BrowserActSuccessor,
+	BrowserActUnavailableSuccessor,
+	BrowserExpectationEvidence,
+	BrowserExpectationStatus,
+	BrowserObservationDiff,
+} from "./translator/types";
 export { createCuaComputerTools } from "./tools";
 export type {
 	BatchDetails,

packages/agent/src/tools.ts

@@ -15,8 +15,11 @@ import {
 	type TSchema,
 } from "@onkernel/cua-ai";
 import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator";
+import type { BrowserActResult } from "./translator/types";
 import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core";
 
+const BROWSER_ACT_DIFF_LINE_LIMIT = 200;
+
 export interface ComputerToolOptions {
 	browser: KernelBrowser;
 	client: Kernel;
@@ -37,6 +40,7 @@ export interface BatchDetails {
 		| { type: "screenshot"; bytes: number }
 		| { type: "cursor_position"; x: number; y: number }
 		| { type: "browser_text"; label: string; bytes: number }
+		| { type: "browser_act"; result: BrowserActResult }
 	>;
 }
 
@@ -170,6 +174,9 @@ async function executeBatchTool(
 			} else if (read.type === "browser_text") {
 				readResults.push({ type: "browser_text", label: read.label, bytes: read.text.length });
 				content.push({ type: "text", text: read.text });
+			} else if (read.type === "browser_act") {
+				readResults.push({ type: "browser_act", result: read.result });
+				content.push({ type: "text", text: formatBrowserActResult(read.result) });
 			} else {
 				readResults.push({ type: "screenshot", bytes: read.data.length });
 				content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType });
@@ -185,7 +192,16 @@ async function executeBatchTool(
 	} catch (err) {
 		throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err });
 	}
-	return { content, details: { statusText: "Actions executed successfully.", readResults } };
+	const actResults = readResults.flatMap((read) => (read.type === "browser_act" ? [read.result] : []));
+	const statusText =
+		actResults.length === 0
+			? "Actions executed successfully."
+			: actResults.some((result) => result.outcome === "didnt")
+				? "Browser actions did not satisfy their expectations."
+				: actResults.every((result) => result.outcome === "worked")
+					? "Browser actions worked."
+					: "Browser action outcome is unknown.";
+	return { content, details: { statusText, readResults } };
 }
 
 async function executeNavigationTool(
@@ -257,6 +273,36 @@ async function executePlaywrightTool(translator: InternalComputerTranslator, par
 	}
 }
 
+function formatBrowserActResult(result: BrowserActResult): string {
+	const lines = [`browser_act outcome: ${result.outcome}`];
+	if (result.stopped_at !== undefined) lines.push(`stopped_at: ${result.stopped_at} (${result.stop_reason ?? "unknown"})`);
+	for (const step of result.steps) {
+		lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.evidence.join("; ")}`);
+		for (const detail of step.expectation?.details ?? []) lines.push(`  ${detail}`);
+	}
+	if (result.final_expectation) {
+		lines.push(`final expectation: ${result.final_expectation.status}`);
+		for (const detail of result.final_expectation.details) lines.push(`  ${detail}`);
+	}
+	if (result.successor.status === "unavailable") {
+		lines.push(`successor unavailable: ${result.successor.error}`);
+	} else {
+		const { diff } = result.successor;
+		lines.push(`successor url: ${result.successor.url}`);
+		lines.push(`successor title: ${result.successor.title}`);
+		lines.push(`successor diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`);
+		if (diff.url) lines.push(`  url: ${diff.url.before} -> ${diff.url.after}`);
+		if (diff.title) lines.push(`  title: ${diff.title.before} -> ${diff.title.after}`);
+		for (const line of diff.added.slice(0, BROWSER_ACT_DIFF_LINE_LIMIT)) lines.push(`  + ${line}`);
+		const remaining = Math.max(0, BROWSER_ACT_DIFF_LINE_LIMIT - diff.added.length);
+		for (const line of diff.removed.slice(0, remaining)) lines.push(`  - ${line}`);
+		const omitted = diff.added.length + diff.removed.length - BROWSER_ACT_DIFF_LINE_LIMIT;
+		if (omitted > 0) lines.push(`  … ${omitted} more diff lines omitted`);
+		lines.push(result.successor.text);
+	}
+	return lines.join("\n");
+}
+
 function formatPlaywrightResult(result: unknown): string {
 	return typeof result === "string" ? result : JSON.stringify(result);
 }