diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 823f318e4..faa3d5ca3 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -135,9 +135,10 @@ export type RotateGestureOptions = DeviceCommandBaseOptions & { export type TransformGestureOptions = DeviceCommandBaseOptions & TransformGestureParams; -export type ScrollOptions = DeviceCommandBaseOptions & { - direction: ScrollInputDirection; - amount?: number; - pixels?: number; - durationMs?: number; -}; +export type ScrollOptions = DeviceCommandBaseOptions & + SettleCommandOptions & { + direction: ScrollInputDirection; + amount?: number; + pixels?: number; + durationMs?: number; + }; diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 74ffaebb8..79edd5916 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -91,7 +91,7 @@ test('usage includes agent workflows, config, environment, and examples footers' assert.match(usageText, /Default app loop: agent-device open /); assert.match( usageText, - /Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close/, + /Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close/, ); assert.match(usageText, /type never accepts --settle/); assert.match(usageText, /explicit success confirmation is visible, stop/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index cecf158a2..ed24aebfc 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -79,7 +79,7 @@ const AGENT_START_LINES = [ // Haiku from 0/2 baseline to 4/4; generic structured-hint recovery passed 8/8 // uncoached output cases versus 7/8 with the longer special-case prose. 'Default app loop: agent-device open -> agent-device snapshot -i -> mutate a current target with --settle -> continue from that settled diff -> agent-device close.', - 'Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', + 'Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', 'Follow structured command hints before choosing a recovery action.', 'Targets are concrete refs or selectors: @e12, label="Query", role=button label="Submit".', 'Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.', diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 80b91799b..e0d5557e9 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -19,17 +19,9 @@ import type { TypeTextOptions, } from '@agent-device/contracts/client'; import type { CommandSchemaOverride } from '../../cli-schema/types.ts'; -import { - commandSupportsSettleObservation, - commandSupportsVerifyEvidence, -} from '../../core/command-descriptor/registry.ts'; -import { - REPEATED_TOUCH_FLAGS, - SELECTOR_SNAPSHOT_FLAGS, - SETTLE_FLAGS, -} from '../cli-grammar/flag-groups.ts'; -import { type FlagKey } from '../cli-grammar/flag-types.ts'; +import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS } from '../cli-grammar/flag-groups.ts'; import { defineExecutableCommand } from '../command-contract.ts'; +import { postActionObservationCliFlags } from '../post-action-observation-surface.ts'; import { commonToClientOptions, toClientElementTarget, @@ -137,20 +129,15 @@ const interactionCliSchemas = { ], }, scroll: { - usageOverride: 'scroll [amount] [--pixels ] [--duration-ms ]', + usageOverride: + 'scroll [amount] [--pixels ] [--duration-ms ] [--settle]', positionalArgs: ['directionOrEdge', 'amount?'], - allowedFlags: ['pixels', 'durationMs'], + allowedFlags: ['pixels', 'durationMs', ...postActionObservationCliFlags('scroll')], }, } as const satisfies Record; type InteractionCommandMetadata = (typeof interactionCommandMetadata)[number]; type InteractionCommandName = InteractionCommandMetadata['name']; -function postActionObservationCliFlags(command: InteractionCommandName): readonly FlagKey[] { - const flags: FlagKey[] = []; - if (commandSupportsVerifyEvidence(command)) flags.push('verify'); - if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS); - return flags; -} const clickCommandDefinition = defineExecutableCommand(metadata('click'), (client, input) => client.interactions.click(toClickOptions(input)), @@ -318,6 +305,7 @@ const scrollCommandFacet = defineCommandFacet({ cliSchema: interactionCliSchemas.scroll, cliReader: interactionCliReaders.scroll, daemonWriter: interactionDaemonWriters.scroll, + cliOutputFormatter: interactionCliOutputFormatters.scroll, }); const getCommandFacet = defineCommandFacet({ diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index eaf41ba68..110eb85ff 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -97,6 +97,7 @@ export const interactionCliReaders = { }, scroll: (positionals, flags) => ({ ...commonInputFromFlags(flags), + ...settleInputFromFlags(flags), direction: readScrollDirection(positionals[0]), amount: optionalCliNumber(positionals[1]), pixels: flags.pixels, diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index b80746fdd..8351960de 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -16,13 +16,9 @@ import { type SwipeGesturePayload, type TransformGesturePayload, } from '@agent-device/contracts/interaction'; -import type { PostActionObservationSupportFor } from '../../core/command-descriptor/post-action-observation.ts'; -import { - commandSupportsSettleObservation, - commandSupportsVerifyEvidence, -} from '../../core/command-descriptor/registry.ts'; import { FIND_LOCATORS } from '@agent-device/selectors'; import { defineCommandMetadata } from '../command-contract.ts'; +import { postActionObservationFields } from '../post-action-observation-surface.ts'; import { booleanField, elementTargetField, @@ -70,7 +66,8 @@ const interactionCommandDescriptions = { focus: 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', - scroll: 'Scroll in a direction, or toward the top/bottom edge of scrollable content.', + scroll: + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Use settle to get the scrolled-into-view diff without a follow-up snapshot.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, editable, selected, focused, or text. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', @@ -80,37 +77,6 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; -const verifyField = () => - booleanField( - 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', - ); - -const settleFields = () => ({ - settle: booleanField( - 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', - ), - settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), - timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), -}); - -type VerifyFieldMap = { verify: ReturnType }; -type SettleFieldMap = ReturnType; -type PostActionObservationFields = - PostActionObservationSupportFor extends 'settle-and-verify' - ? VerifyFieldMap & SettleFieldMap - : PostActionObservationSupportFor extends 'settle' - ? SettleFieldMap - : {}; - -function postActionObservationFields( - command: TName, -): PostActionObservationFields { - return { - ...(commandSupportsVerifyEvidence(command) ? { verify: verifyField() } : {}), - ...(commandSupportsSettleObservation(command) ? settleFields() : {}), - } as PostActionObservationFields; -} - const clickFields = { target: requiredField(interactionTargetField()), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), @@ -170,6 +136,7 @@ const scrollFields = { min: 0, max: SCROLL_DURATION_MAX_MS, }), + ...postActionObservationFields('scroll'), }; // #1271 stage 2 (ADR 0012 amendment): `get`/`is`/`find` are observation-only, diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index 4c9ad2ef2..db2799f60 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -13,6 +13,9 @@ const formatFill = (result: Record) => const formatLongPress = (result: Record) => interactionCliOutputFormatters.longpress({ input: {}, result }); +const formatScroll = (result: Record) => + interactionCliOutputFormatters.scroll({ input: {}, result }); + describe('find CLI output', () => { test('click prints the same success line as a direct press', () => { const output = formatFind({ @@ -275,3 +278,39 @@ describe('longpress CLI output', () => { ); }); }); + +describe('scroll CLI output', () => { + test('prints the scroll message alone without a settle observation', () => { + const output = formatScroll({ message: 'Scrolled down' }); + + expect(output.text).toBe('Scrolled down'); + }); + + // #1638: the generic-route settle renders exactly like the touch commands. + test('appends settle verdict and diff lines when present', () => { + const output = formatScroll({ + message: 'Scrolled down', + settle: { + settled: true, + waitedMs: 400, + refsGeneration: 7, + diff: { + summary: { additions: 1, removals: 1, unchanged: 9 }, + lines: [ + { kind: 'removed', text: '@e2 [button] "Load more"' }, + { kind: 'added', text: '@e2 [button] "Next page"', ref: 'e2' }, + ], + }, + }, + }); + + expect(output.text).toBe( + [ + 'Scrolled down', + 'settled after 400ms: +1 -1 (~9 unchanged)', + '- @e2 [button] "Load more"', + '+ @e2 [button] "Next page"', + ].join('\n'), + ); + }); +}); diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 0ab316db8..28b9360c5 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -4,6 +4,7 @@ import type { CliOutput } from '../command-contract.ts'; import { displayLabel, formatRole } from '../../snapshot/snapshot-lines.ts'; import { readCommandMessage } from '../../utils/success-text.ts'; import { messageCliOutput, resultOutput, type CliOutputFormatter } from '../output-common.ts'; +import { appendResponseNotes, pinnedRefText } from '../settle-output.ts'; function getCliOutput(params: { result: CommandRequestResult; format?: string }): CliOutput { const data = params.result as Record; @@ -16,17 +17,6 @@ function getCliOutput(params: { result: CommandRequestResult; format?: string }) return defaultCommandCliOutput(data); } -// ADR 0014: a reusable ref in a PARTIAL result renders in ready-to-copy -// `@eN~s` form so a human CLI caller can paste it into the next -// mutation without a separate pin step. A mutating result carries no -// `refsGeneration`, so its acted ref is never pinned. -function pinnedRefText(ref: unknown, refsGeneration: unknown): string | undefined { - if (typeof ref !== 'string' || ref.length === 0) return undefined; - if (typeof refsGeneration !== 'number') return undefined; - const body = ref.startsWith('@') ? ref.slice(1) : ref; - return `@${body}~s${refsGeneration}`; -} - function findCliOutput(result: CommandRequestResult): CliOutput { const data = result as Record; // Interactive find actions (click/fill/focus/type) carry the same success message as @@ -90,85 +80,14 @@ function messageWithSettleCliOutput(result: CommandRequestResult): CliOutput { return { data: output.data, text: appendResponseNotes(output.text, data) }; } -function appendResponseNotes( - text: string | null | undefined, - data: Record, -): string { - const warning = typeof data.warning === 'string' ? `\nWarning: ${data.warning}` : ''; - return `${text ?? ''}${warning}${formatSettleText(data.settle)}`; -} - -type SettleTextView = { - settled?: boolean; - waitedMs?: number; - hint?: string; - diff?: { - summary?: { additions?: number; removals?: number; unchanged?: number }; - lines?: Array<{ kind?: string; text?: string }>; - truncated?: boolean; - }; - tail?: Array<{ ref?: string; role?: string; label?: string }>; - tailTruncated?: boolean; - refsGeneration?: number; -}; - -/** - * Compact `--settle` (#1101) rendering appended to the tap line: the verdict, - * the changed-count summary, and the changed lines themselves (the payload the - * agent acts on). Empty for non-settle responses. - */ -function formatSettleText(settle: unknown): string { - if (!settle || typeof settle !== 'object') return ''; - const view = settle as SettleTextView; - const parts = [ - formatSettleVerdict(view), - ...formatSettleDiffLines(view.diff), - ...formatSettleTailLines(view), - ]; - if (view.hint) parts.push(`hint: ${view.hint}`); - return `\n${parts.join('\n')}`; -} - -function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { - const lines = (diff?.lines ?? []).map( - (line) => `${line.kind === 'removed' ? '-' : '+'} ${line.text ?? ''}`, - ); - if (diff?.truncated) lines.push('… changed lines truncated'); - return lines; -} - -// Unchanged interactive tail: only present when the diff's added lines -// carried zero refs (modal-dismiss/toast-only diff), so the settled tree's -// remaining actionable elements would otherwise be invisible. -function formatSettleTailLines(view: SettleTextView): string[] { - const tail = view.tail ?? []; - if (tail.length === 0) return []; - const lines = [`unchanged interactive (${tail.length}):`]; - for (const entry of tail) { - const label = entry.label ? ` "${entry.label}"` : ''; - // ADR 0014: the settled tail refs are reusable, so render them pinned when - // the settle response carried its generation. - const ref = pinnedRefText(entry.ref, view.refsGeneration) ?? `@${entry.ref ?? ''}`; - lines.push(`= ${ref} [${entry.role ?? ''}]${label}`); - } - if (view.tailTruncated) { - lines.push('… more interactive elements not shown, use snapshot -i'); - } - return lines; -} - -function formatSettleVerdict(view: SettleTextView): string { - const verdict = view.settled === true ? 'settled' : 'not settled'; - const summary = view.diff?.summary; - if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; - return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; -} - export const interactionCliOutputFormatters = { click: resultOutput(tapCliOutput), press: resultOutput(tapCliOutput), fill: resultOutput(messageWithSettleCliOutput), longpress: resultOutput(messageWithSettleCliOutput), + // #1638: scroll takes --settle on the generic route, so its line renders the + // settled diff exactly like the touch commands. + scroll: resultOutput(messageWithSettleCliOutput), get: ({ input, result }) => getCliOutput({ result: result as CommandRequestResult, diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index f46bbc532..17a3c2b74 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -53,6 +53,8 @@ import { type GestureCommandOptions, type GestureCommandResult, } from './gesture-command.ts'; +import { settleObservationCommand, type SettleObservationCommandOptions } from './settle.ts'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; export type SelectorCommands = { find: RuntimeCommand; @@ -78,6 +80,12 @@ export type InteractionCommands = { longPress: RuntimeCommand; scroll: RuntimeCommand; gesture: RuntimeCommand; + /** + * `--settle` (#1101) for a caller that resolved no target — the daemon's + * generic scroll/back route (#1638). Every other settle-carrying command gets + * its observation folded into its own result. + */ + observeSettle: RuntimeCommand; }; export type BoundSelectorCommands = { @@ -135,6 +143,7 @@ export type BoundInteractionCommands = { ) => Promise; scroll: BoundRuntimeCommand; gesture: BoundRuntimeCommand; + observeSettle: BoundRuntimeCommand; }; export const selectorCommands: SelectorCommands = { @@ -158,6 +167,7 @@ export const interactionCommands: InteractionCommands = { longPress: longPressCommand, scroll: scrollCommand, gesture: gestureCommand, + observeSettle: settleObservationCommand, }; export function bindSelectorCommands(runtime: AgentDeviceRuntime): BoundSelectorCommands { @@ -188,6 +198,7 @@ export function bindInteractionCommands(runtime: AgentDeviceRuntime): BoundInter interactionCommands.longPress(runtime, { ...options, target }), scroll: (options) => interactionCommands.scroll(runtime, options), gesture: (options) => interactionCommands.gesture(runtime, options), + observeSettle: (options) => interactionCommands.observeSettle(runtime, options), }; } diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 0d2abaa27..d6e7f18f5 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -16,6 +16,7 @@ import type { SettleParams, SettleTailEntry, } from '@agent-device/contracts/interaction'; +import type { RuntimeCommand } from '../../runtime-types.ts'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { DEFAULT_STABLE_QUIET_MS, @@ -62,6 +63,55 @@ export async function settleAfterInteraction( runtime: AgentDeviceRuntime, options: CommandContext, params: SettleParams & { resolved: ResolvedInteractionTarget }, +): Promise { + const { resolved, ...settleParams } = params; + return await settleAfterAction(runtime, options, { + ...settleParams, + baselineNodes: resolveBaselineNodes(resolved), + ...(resolved.point ? { actionPoint: resolved.point } : {}), + }); +} + +export type SettleObservationCommandOptions = CommandContext & + SettleParams & { + /** The pre-action tree the settled capture is diffed against. */ + baselineNodes: SnapshotNode[]; + }; + +/** + * The target-less settle as a runtime command (#1638), so a caller that never + * resolved a target — the daemon's generic scroll/back route — reaches the same + * observation through the same runtime seam every other settle-carrying command + * uses, instead of reaching into the engine directly. + * + * Returns only the observation: with no resolved target there is no `--verify` + * evidence to pair the settled nodes with. + */ +export const settleObservationCommand: RuntimeCommand< + SettleObservationCommandOptions, + SettleObservation +> = async (runtime, options) => { + const { baselineNodes, ...settle } = options; + const outcome = await settleAfterAction(runtime, options, { ...settle, baselineNodes }); + return outcome.observation; +}; + +/** + * The target-less settle (#1638): same stable-capture loop, storage rules and + * hints as {@link settleAfterInteraction}, but the caller supplies the diff + * baseline instead of it coming from a freshly resolved target, and there is + * no action point to recognize self-echo added lines by. + * + * The generic route (scroll/back) passes the session's STORED pre-action + * snapshot nodes as the baseline. That tree can be several commands older than + * the action — which is the honest reading of the #1101 contract ("the settled + * diff vs the pre-action tree"), but is a weaker baseline than press's + * freshly-resolved one. + */ +async function settleAfterAction( + runtime: AgentDeviceRuntime, + options: CommandContext, + params: SettleParams & { baselineNodes: SnapshotNode[]; actionPoint?: Point }, ): Promise { const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; @@ -72,44 +122,7 @@ export async function settleAfterInteraction( timeoutMs, resetBudgetOnPrivateAxRecovery: true, }); - const observation: SettleObservation = { - ...base, - settled: outcome.settled, - waitedMs: outcome.waitedMs, - captures: outcome.captures, - }; - if (!outcome.lastCapture) { - return { - observation: { - ...observation, - hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT, - }, - }; - } - const { stored, session } = await storeSettledSnapshot(runtime, options, outcome.lastCapture); - const settledNodes = outcome.lastCapture.snapshot.nodes; - return { - observation: { - ...observation, - // The diff (with its added-line refs) is only attached when the settled - // tree actually became the stored session snapshot: those refs must be - // valid against the tree the next @ref command resolves on. The daemon - // treats `diff` presence as "this response issues refs". Unsettled - // captures are intentionally diff-less: they are not a stable - // observation, so surfacing refs would invite agents to act on - // advisory state. - ...(outcome.settled && stored - ? buildSettleDiffAndTail( - resolveBaselineNodes(params.resolved), - settledNodes, - params.resolved.point, - session?.appBundleId, - ) - : {}), - ...resolveSettleHint(outcome, stored, settledNodes.length), - }, - settledNodes, - }; + return await buildSettleOutcome(runtime, options, { ...params, base, outcome }); } catch (error) { // Never fail the action over the observation: report that settling itself // broke and let the caller fall back to an explicit snapshot. @@ -122,6 +135,60 @@ export async function settleAfterInteraction( } } +type StableCaptureOutcome = Awaited>; + +/** Turns a completed stable-capture loop into the settle observation it reports. */ +async function buildSettleOutcome( + runtime: AgentDeviceRuntime, + options: CommandContext, + params: { + baselineNodes: SnapshotNode[]; + actionPoint?: Point; + base: SettleObservation; + outcome: StableCaptureOutcome; + }, +): Promise { + const { base, outcome } = params; + const observation: SettleObservation = { + ...base, + settled: outcome.settled, + waitedMs: outcome.waitedMs, + captures: outcome.captures, + }; + if (!outcome.lastCapture) { + return { + observation: { + ...observation, + hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT, + }, + }; + } + const { stored, session } = await storeSettledSnapshot(runtime, options, outcome.lastCapture); + const settledNodes = outcome.lastCapture.snapshot.nodes; + return { + observation: { + ...observation, + // The diff (with its added-line refs) is only attached when the settled + // tree actually became the stored session snapshot: those refs must be + // valid against the tree the next @ref command resolves on. The daemon + // treats `diff` presence as "this response issues refs". Unsettled + // captures are intentionally diff-less: they are not a stable + // observation, so surfacing refs would invite agents to act on + // advisory state. + ...(outcome.settled && stored + ? buildSettleDiffAndTail( + params.baselineNodes, + settledNodes, + params.actionPoint, + session?.appBundleId, + ) + : {}), + ...resolveSettleHint(outcome, stored, settledNodes.length), + }, + settledNodes, + }; +} + /** * `--settle --verify` composition: the settle loop's final capture doubles as * the verify evidence source, so the pair costs zero extra captures. Without a diff --git a/src/commands/post-action-observation-surface.ts b/src/commands/post-action-observation-surface.ts new file mode 100644 index 000000000..73e78ce5f --- /dev/null +++ b/src/commands/post-action-observation-surface.ts @@ -0,0 +1,58 @@ +import type { PostActionObservationSupportFor } from '../core/command-descriptor/post-action-observation.ts'; +import { + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, +} from '../core/command-descriptor/registry.ts'; +import { SETTLE_FLAGS } from './cli-grammar/flag-groups.ts'; +import type { FlagKey } from './cli-grammar/flag-types.ts'; +import { booleanField, integerField } from './command-input.ts'; + +/** + * The command-surface projection of the descriptor post-action observation + * trait (src/core/command-descriptor/post-action-observation.ts): the CLI flags + * a settle/verify-capable command accepts, and the input fields its MCP tool + * and SDK options expose. Both derive from the trait map, so granting a command + * `--settle` is one registry edit rather than a hunt through the per-family + * grammar files. + * + * It lives outside `interaction/` because the trait is not an interaction-family + * property: `back` (src/commands/system/index.ts) carries it too (#1638). + */ + +const verifyField = () => + booleanField( + 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', + ); + +const settleFields = () => ({ + settle: booleanField( + 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', + ), + settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), + timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), +}); + +type VerifyFieldMap = { verify: ReturnType }; +type SettleFieldMap = ReturnType; +export type PostActionObservationFields = + PostActionObservationSupportFor extends 'settle-and-verify' + ? VerifyFieldMap & SettleFieldMap + : PostActionObservationSupportFor extends 'settle' + ? SettleFieldMap + : {}; + +export function postActionObservationFields( + command: TName, +): PostActionObservationFields { + return { + ...(commandSupportsVerifyEvidence(command) ? { verify: verifyField() } : {}), + ...(commandSupportsSettleObservation(command) ? settleFields() : {}), + } as PostActionObservationFields; +} + +export function postActionObservationCliFlags(command: string): readonly FlagKey[] { + const flags: FlagKey[] = []; + if (commandSupportsVerifyEvidence(command)) flags.push('verify'); + if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS); + return flags; +} diff --git a/src/commands/settle-output.ts b/src/commands/settle-output.ts new file mode 100644 index 000000000..cfe04048d --- /dev/null +++ b/src/commands/settle-output.ts @@ -0,0 +1,95 @@ +/** + * CLI rendering for the `--settle` (#1101) observation. Every command that can + * attach one renders it identically — the touch commands, and the generic-route + * `scroll`/`back` (#1638) — so the verdict/diff/tail shape an agent learns from + * one command is the shape it gets from all of them. + */ + +// ADR 0014: a reusable ref in a PARTIAL result renders in ready-to-copy +// `@eN~s` form so a human CLI caller can paste it into the next +// mutation without a separate pin step. A mutating result carries no +// `refsGeneration`, so its acted ref is never pinned. +export function pinnedRefText(ref: unknown, refsGeneration: unknown): string | undefined { + if (typeof ref !== 'string' || ref.length === 0) return undefined; + if (typeof refsGeneration !== 'number') return undefined; + const body = ref.startsWith('@') ? ref.slice(1) : ref; + return `@${body}~s${refsGeneration}`; +} + +type SettleTextView = { + settled?: boolean; + waitedMs?: number; + hint?: string; + diff?: { + summary?: { additions?: number; removals?: number; unchanged?: number }; + lines?: Array<{ kind?: string; text?: string }>; + truncated?: boolean; + }; + tail?: Array<{ ref?: string; role?: string; label?: string }>; + tailTruncated?: boolean; + refsGeneration?: number; +}; + +/** + * Appends the response's advisory notes — the warning line, then the compact + * settle rendering — to a command's own success text. + */ +export function appendResponseNotes( + text: string | null | undefined, + data: Record, +): string { + const warning = typeof data.warning === 'string' ? `\nWarning: ${data.warning}` : ''; + return `${text ?? ''}${warning}${formatSettleText(data.settle)}`; +} + +/** + * Compact `--settle` (#1101) rendering appended to the action line: the verdict, + * the changed-count summary, and the changed lines themselves (the payload the + * agent acts on). Empty for non-settle responses. + */ +function formatSettleText(settle: unknown): string { + if (!settle || typeof settle !== 'object') return ''; + const view = settle as SettleTextView; + const parts = [ + formatSettleVerdict(view), + ...formatSettleDiffLines(view.diff), + ...formatSettleTailLines(view), + ]; + if (view.hint) parts.push(`hint: ${view.hint}`); + return `\n${parts.join('\n')}`; +} + +function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { + const lines = (diff?.lines ?? []).map( + (line) => `${line.kind === 'removed' ? '-' : '+'} ${line.text ?? ''}`, + ); + if (diff?.truncated) lines.push('… changed lines truncated'); + return lines; +} + +// Unchanged interactive tail: only present when the diff's added lines +// carried zero refs (modal-dismiss/toast-only diff), so the settled tree's +// remaining actionable elements would otherwise be invisible. +function formatSettleTailLines(view: SettleTextView): string[] { + const tail = view.tail ?? []; + if (tail.length === 0) return []; + const lines = [`unchanged interactive (${tail.length}):`]; + for (const entry of tail) { + const label = entry.label ? ` "${entry.label}"` : ''; + // ADR 0014: the settled tail refs are reusable, so render them pinned when + // the settle response carried its generation. + const ref = pinnedRefText(entry.ref, view.refsGeneration) ?? `@${entry.ref ?? ''}`; + lines.push(`= ${ref} [${entry.role ?? ''}]${label}`); + } + if (view.tailTruncated) { + lines.push('… more interactive elements not shown, use snapshot -i'); + } + return lines; +} + +function formatSettleVerdict(view: SettleTextView): string { + const verdict = view.settled === true ? 'settled' : 'not settled'; + const summary = view.diff?.summary; + if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; + return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; +} diff --git a/src/commands/system/index.test.ts b/src/commands/system/index.test.ts index 14cc3b7cc..7f3a1160d 100644 --- a/src/commands/system/index.test.ts +++ b/src/commands/system/index.test.ts @@ -27,6 +27,7 @@ import { tvRemoteDaemonWriter, systemCommandFamily, } from './index.ts'; +import { systemCliOutputFormatters } from './output.ts'; function flags(overrides: Partial = {}): CliFlags { return overrides as CliFlags; @@ -118,6 +119,50 @@ describe('system command interface', () => { ).toBeUndefined(); }); + // #1638: --settle has to survive BOTH back seams — the reader that builds the + // input and the writer that turns it into daemon request options. + test('back reader and writer carry the settle request through to daemon flags', () => { + const input = backCliReader([], flags({ settle: true, settleQuietMs: 250, timeoutMs: 8_000 })); + expect(input).toMatchObject({ settle: true, settleQuietMs: 250, timeoutMs: 8_000 }); + expect(backDaemonWriter(input).options).toMatchObject({ + settle: true, + settleQuietMs: 250, + timeoutMs: 8_000, + }); + expect(backCliReader([], flags()).settle).toBeUndefined(); + }); + + test('back CLI output renders the settled observation', () => { + const output = systemCliOutputFormatters.back({ + input: {}, + result: { + action: 'back', + mode: 'in-app', + message: 'Back', + settle: { + settled: true, + waitedMs: 300, + diff: { + summary: { additions: 1, removals: 2, unchanged: 5 }, + lines: [ + { kind: 'removed', text: '@e9 [button] "Save"' }, + { kind: 'added', text: '@e3 [button] "Edit"', ref: 'e3' }, + ], + }, + }, + }, + }); + + expect(output.text).toBe( + [ + 'Back', + 'settled after 300ms: +1 -2 (~5 unchanged)', + '- @e9 [button] "Save"', + '+ @e3 [button] "Edit"', + ].join('\n'), + ); + }); + test('orientation reader and writer normalize orientation', () => { expect(orientationCliReader(['left'], flags())).toMatchObject({ orientation: 'landscape-left', diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 29bfa38f6..a92018543 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -16,6 +16,7 @@ import { optionalString, request, requiredDaemonString, + settleInputFromFlags, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import { defineExecutableCommand } from '../command-contract.ts'; @@ -32,6 +33,10 @@ import { projectCommandOutputSchemas, } from '../family/types.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { + postActionObservationCliFlags, + postActionObservationFields, +} from '../post-action-observation-surface.ts'; import { NAVIGATION_COMMAND_PROJECTIONS } from './navigation-projection.ts'; import { systemCliOutputFormatters } from './output.ts'; @@ -71,6 +76,7 @@ const appStateCommandMetadata = defineFieldCommandMetadata( const backCommandMetadata = defineFieldCommandMetadata(BACK_COMMAND_NAME, backCommandDescription, { mode: enumField(BACK_MODES), + ...postActionObservationFields(BACK_COMMAND_NAME), }); const homeCommandMetadata = defineFieldCommandMetadata( @@ -172,8 +178,8 @@ const tvRemoteCommandDefinition = defineExecutableCommand( const appStateCliSchema = {} as const satisfies CommandSchemaOverride; const backCliSchema = { - usageOverride: 'back [--in-app|--system]', - allowedFlags: ['backMode'], + usageOverride: 'back [--in-app|--system] [--settle]', + allowedFlags: ['backMode', ...postActionObservationCliFlags(BACK_COMMAND_NAME)], } as const satisfies CommandSchemaOverride; const homeCliSchema = {} as const satisfies CommandSchemaOverride; @@ -210,6 +216,7 @@ export const appSwitcherCliReader: CliReader = (_positionals, flags) => commonIn export const backCliReader: CliReader = (_positionals, flags) => ({ ...commonInputFromFlags(flags), + ...settleInputFromFlags(flags), mode: flags.backMode, }); diff --git a/src/commands/system/navigation-projection.ts b/src/commands/system/navigation-projection.ts index 9102093b1..5de98e7e4 100644 --- a/src/commands/system/navigation-projection.ts +++ b/src/commands/system/navigation-projection.ts @@ -1,3 +1,4 @@ +import type { SettleCommandOptions } from '@agent-device/contracts/client'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { BACK_MODES, @@ -39,7 +40,16 @@ function defineNavigationCommandProjection< } export const NAVIGATION_COMMAND_PROJECTIONS = { - back: defineNavigationCommandProjection<{ mode?: BackMode }, BackCommandResult, false, 'back'>({ + // #1638: `back` carries the descriptor post-action observation trait, so its + // options accept `--settle`. The settled observation itself rides the payload + // additively — these schemas are never strict, exactly so an opt-in field can + // (like `cost`) validate without every projection re-declaring it. + back: defineNavigationCommandProjection< + { mode?: BackMode } & SettleCommandOptions, + BackCommandResult, + false, + 'back' + >({ clientMethod: 'back', outputSchema: { type: 'object', diff --git a/src/commands/system/output.ts b/src/commands/system/output.ts index 6ba530640..79731e83c 100644 --- a/src/commands/system/output.ts +++ b/src/commands/system/output.ts @@ -10,6 +10,7 @@ import { resultOutput, type CliOutputFormatter, } from '../output-common.ts'; +import { appendResponseNotes } from '../settle-output.ts'; function appStateCliOutput(result: AppStateCommandResult): CliOutput { return { @@ -40,9 +41,16 @@ function clipboardCliOutput(result: ClipboardCommandResult): CliOutput { return messageCliOutput(result); } +// #1638: back takes --settle on the generic route, so its line renders the +// settled diff exactly like the touch commands. +function backCliOutput(result: Record): CliOutput { + const output = messageCliOutput(result); + return { data: output.data, text: appendResponseNotes(output.text, result) }; +} + export const systemCliOutputFormatters = { appstate: resultOutput(appStateCliOutput), - back: messageOutput, + back: resultOutput(backCliOutput), home: messageOutput, orientation: messageOutput, 'app-switcher': messageOutput, diff --git a/src/core/command-descriptor/__tests__/post-action-observation.test.ts b/src/core/command-descriptor/__tests__/post-action-observation.test.ts index ee0521204..60c2bf511 100644 --- a/src/core/command-descriptor/__tests__/post-action-observation.test.ts +++ b/src/core/command-descriptor/__tests__/post-action-observation.test.ts @@ -11,10 +11,12 @@ import { } from '../registry.ts'; const SETTLE_OBSERVATION_COMMANDS = [ + PUBLIC_COMMANDS.back, PUBLIC_COMMANDS.click, PUBLIC_COMMANDS.fill, PUBLIC_COMMANDS.longPress, PUBLIC_COMMANDS.press, + PUBLIC_COMMANDS.scroll, ] as const; test('post-action observation descriptor traits are the source for settle command support', () => { @@ -31,6 +33,12 @@ test('post-action observation descriptor traits are the source for settle comman assert.equal(resolveCommandPostActionObservationSupport('fill'), 'settle-and-verify'); assert.equal(resolveCommandPostActionObservationSupport('longpress'), 'settle'); assert.equal(commandSupportsVerifyEvidence('longpress'), false); + // #1638: the generic-route pair resolves no target, so there is no pre-action + // node to digest into evidence — settle without verify. + assert.equal(resolveCommandPostActionObservationSupport('scroll'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('scroll'), false); + assert.equal(resolveCommandPostActionObservationSupport('back'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('back'), false); }); test('post-action observation CLI flags follow descriptor traits', () => { diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index 3d9bba776..959f48aa7 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -68,10 +68,14 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // Interaction commands joined in #1105: their target resolution runs the // same capture as snapshot, and resetting the daemon on a wedged capture // destroyed healthy app sessions. + // `back`/`scroll` joined in #1638: `--settle` gives them the same capture + // hang mode, and the policy cannot be flag-conditional — preserving is the + // safe side of that choice (a reset loses every session on the daemon). const preserving = commandDescriptors .filter((descriptor) => descriptor.timeoutPolicy.onTimeout === 'preserve-daemon') .map((descriptor) => descriptor.name); assert.deepEqual(preserving.sort(), [ + 'back', 'click', 'fill', 'find', @@ -79,6 +83,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { 'is', 'longpress', 'press', + 'scroll', 'snapshot', 'type', 'wait', diff --git a/src/core/command-descriptor/post-action-observation.ts b/src/core/command-descriptor/post-action-observation.ts index 355d9a741..48d2598c9 100644 --- a/src/core/command-descriptor/post-action-observation.ts +++ b/src/core/command-descriptor/post-action-observation.ts @@ -1,10 +1,23 @@ export type PostActionObservationSupport = 'settle' | 'settle-and-verify'; +/** + * The single source for which commands accept `--settle`/`--verify` (#1101, + * #1047). Every downstream seam — CLI flag grammar, MCP/SDK input fields, + * daemon flag guards, timeout policy, dispatch wiring — derives its answer + * from this map rather than repeating a command list. + * + * `scroll` and `back` (#1638) carry `settle` only: they resolve no target, so + * there is no pre-action node to digest into `--verify` evidence. Their + * settled diff is taken against the STORED pre-action snapshot, which the + * generic route holds instead of a freshly resolved baseline. + */ const POST_ACTION_OBSERVATION_BY_COMMAND = { click: 'settle-and-verify', press: 'settle-and-verify', fill: 'settle-and-verify', longpress: 'settle', + scroll: 'settle', + back: 'settle', } as const satisfies Record; export type PostActionObservationCommandName = keyof typeof POST_ACTION_OBSERVATION_BY_COMMAND; diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 1889db663..7bfe70fc6 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -287,10 +287,24 @@ const FILL_INTERACTION_RESPONSE_DATA_TRANSFORM = { }, } as const satisfies CommandResponseDataTransform; -function interactionTimeoutPolicy(command: string): CommandTimeoutPolicy { +/** + * Timeout policy for a command that MAY carry the post-action observation + * trait: the settle policy when the observation map lists it, the caller's + * baseline otherwise. Keeping the map the enumerator means adding `--settle` + * to a command cannot leave its request envelope too narrow for the settle + * wait it now accepts. + */ +function postActionObservationTimeoutPolicy( + command: string, + withoutObservation: CommandTimeoutPolicy, +): CommandTimeoutPolicy { return resolvePostActionObservationSupport(command) !== undefined ? SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY - : PRESERVE_DAEMON_TIMEOUT_POLICY; + : withoutObservation; +} + +function interactionTimeoutPolicy(command: string): CommandTimeoutPolicy { + return postActionObservationTimeoutPolicy(command, PRESERVE_DAEMON_TIMEOUT_POLICY); } function postActionObservation(command: string): PostActionObservationSupport { @@ -1030,6 +1044,14 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.capability, vega: VEGA_VVD, }, + // #1638: back-then-observe in one round trip. The settle wait runs on the + // generic route (src/daemon/request-generic-dispatch.ts), after Android's + // blocking-dialog postflight. + timeoutPolicy: postActionObservationTimeoutPolicy( + 'back', + GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.timeoutPolicy, + ), + postActionObservation: postActionObservation('back'), }, { name: 'gesture', @@ -1103,6 +1125,12 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), catalog: { group: 'public' }, ...GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS, + // #1638: scroll-then-observe in one round trip; see `back` above. + timeoutPolicy: postActionObservationTimeoutPolicy( + 'scroll', + GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.timeoutPolicy, + ), + postActionObservation: postActionObservation('scroll'), }, { name: 'swipe', diff --git a/src/daemon/__tests__/generic-settle.test.ts b/src/daemon/__tests__/generic-settle.test.ts new file mode 100644 index 000000000..484ce9fe0 --- /dev/null +++ b/src/daemon/__tests__/generic-settle.test.ts @@ -0,0 +1,306 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SnapshotBackend } from '@agent-device/kernel/snapshot'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import type { SessionStore } from '../session-store.ts'; +import type { DaemonResponse, SessionState } from '../types.ts'; +import { activateCompleteRefFrame } from '../ref-frame.ts'; +import { isPostGestureStabilizationPending } from '../deferred-interaction-outcome.ts'; +import { buildSnapshotState } from '../handlers/snapshot-capture.ts'; +import { setSessionSnapshot } from '../session-snapshot.ts'; + +// #1638 `--settle` on the generic route (scroll/back): the settled diff is +// taken against the session's STORED pre-action tree, rides the response as +// `data.settle`, and is ref-issuing exactly like the touch route. Quiet windows +// are tuned down (settleQuietMs 25) so no test waits real time. + +vi.mock('../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; +}); + +vi.mock('../handlers/interaction-snapshot.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, captureSnapshotForSession: vi.fn() }; +}); + +import { dispatchCommand } from '../../core/dispatch.ts'; +import { captureSnapshotForSession } from '../handlers/interaction-snapshot.ts'; +import { dispatchGenericCommand } from '../request-generic-dispatch.ts'; + +const mockDispatch = vi.mocked(dispatchCommand); +const mockCapture = vi.mocked(captureSnapshotForSession); + +const BEFORE_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Load more', + rect: { x: 10, y: 700, width: 120, height: 44 }, + hittable: true, + }, +]; + +const AFTER_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Next page', + rect: { x: 10, y: 700, width: 120, height: 44 }, + hittable: true, + }, +]; + +type SettlePayload = { + settled: boolean; + captures: number; + quietMs: number; + timeoutMs: number; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + hint?: string; +}; + +const contextFromFlags = () => ({}) as never; + +/** Every settle capture observes the post-action tree. */ +function captureNodes(nodes: typeof BEFORE_NODES, onCapture?: (session: SessionState) => void) { + mockCapture.mockImplementation(async (session, flags, sessionStore) => { + onCapture?.(session); + const snapshot = buildSnapshotState( + { nodes, backend: 'xctest' as SnapshotBackend }, + (flags ?? {}) as CommandFlags, + ); + setSessionSnapshot(session, snapshot); + sessionStore.set(session.name, session); + return snapshot; + }); +} + +function seedSession(sessionName: string, sessionStore: SessionStore): SessionState { + const session = makeIosSession(sessionName); + setSessionSnapshot(session, buildSnapshotState({ nodes: BEFORE_NODES, backend: 'xctest' }, {})); + // The seed emulates a snapshot response that issued these refs (ADR 0014). + activateCompleteRefFrame(session); + sessionStore.set(sessionName, session); + return session; +} + +async function runGeneric(params: { + sessionName: string; + sessionStore: SessionStore; + command: string; + positionals?: string[]; + flags?: CommandFlags; +}): Promise { + const session = params.sessionStore.get(params.sessionName) as SessionState; + return await dispatchGenericCommand({ + req: { + token: 't', + session: params.sessionName, + command: params.command, + positionals: params.positionals ?? [], + ...(params.flags ? { flags: params.flags } : {}), + }, + session, + sessionName: params.sessionName, + logPath: '', + sessionStore: params.sessionStore, + contextFromFlags, + }); +} + +function expectOkData(response: DaemonResponse): Record { + expect(response.ok).toBe(true); + if (!response.ok) throw new Error('expected an ok daemon response'); + return (response.data ?? {}) as Record; +} + +const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 } satisfies CommandFlags; + +beforeEach(() => { + mockDispatch.mockReset(); + mockDispatch.mockResolvedValue({}); + mockCapture.mockReset(); +}); + +test('scroll --settle diffs the settled tree against the stored pre-action snapshot', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-scroll'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(25); + expect(settle.timeoutMs).toBe(2_000); + // Baseline is the STORED pre-action tree, not a fresh resolution capture. + expect(settle.diff?.summary).toEqual({ additions: 1, removals: 1, unchanged: 1 }); + expect(settle.diff?.lines.find((line) => line.kind === 'added')).toEqual({ + kind: 'added', + text: expect.stringContaining('Next page'), + ref: 'e2', + }); + // The device action itself still ran, once. + expect(mockDispatch.mock.calls.filter(([, command]) => command === 'scroll')).toHaveLength(1); +}); + +test('scroll --settle is ref-issuing: partial frame plus the stored tree generation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-refs'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + const settle = expectOkData(response).settle as SettlePayload; + const session = sessionStore.get(sessionName) as SessionState; + expect(session.refFrameState).toBe('active'); + expect(settle.refsGeneration).toBe(session.snapshotGeneration); + expect(session.snapshot?.nodes.some((node) => node.label === 'Next page')).toBe(true); +}); + +test('back --settle carries the settled observation on the navigation payload', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-back'; + seedSession(sessionName, sessionStore); + mockDispatch.mockImplementation(async (_device, command) => + command === 'back' ? { action: 'back', mode: 'in-app', message: 'Back' } : {}, + ); + captureNodes(AFTER_NODES); + + const data = expectOkData( + await runGeneric({ + sessionName, + sessionStore, + command: 'back', + flags: { ...SETTLE_FLAGS, backMode: 'in-app' }, + }), + ); + + // The command's own payload survives alongside the additive observation. + expect(data.action).toBe('back'); + expect(data.message).toBe('Back'); + expect((data.settle as SettlePayload).settled).toBe(true); +}); + +test('the settle capture runs after the post-gesture stabilization marker is placed', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-order'; + seedSession(sessionName, sessionStore); + // #1542 ordering: settle's FIRST capture must be the one that folds the + // pending stabilization, so the marker has to exist by the time it runs. + const pendingAtCapture: boolean[] = []; + captureNodes(AFTER_NODES, (session) => { + pendingAtCapture.push(isPostGestureStabilizationPending(session)); + }); + + await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + expect(pendingAtCapture[0]).toBe(true); +}); + +test('scroll without --settle observes nothing', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-no-settle'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const data = expectOkData( + await runGeneric({ sessionName, sessionStore, command: 'scroll', positionals: ['down'] }), + ); + + expect(data.settle).toBeUndefined(); + expect(mockCapture).not.toHaveBeenCalled(); +}); + +test('a generic command without the observation trait ignores a stray settle flag', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-untraited'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + // `home` is a generic leaf with no post-action observation trait. The CLI + // schema already refuses --settle for it; the daemon must not observe either. + const data = expectOkData( + await runGeneric({ sessionName, sessionStore, command: 'home', flags: { ...SETTLE_FLAGS } }), + ); + + expect(data.settle).toBeUndefined(); + expect(mockCapture).not.toHaveBeenCalled(); +}); + +test('scroll --settle-quiet without --settle is rejected before the device action', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-orphan'; + seedSession(sessionName, sessionStore); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settleQuietMs: 25 }, + }); + + expect(response.ok).toBe(false); + if (response.ok) throw new Error('expected a rejected daemon response'); + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.message).toMatch(/--settle-quiet requires --settle/); + expect(mockDispatch).not.toHaveBeenCalled(); +}); + +test('a failed settle observation never fails the action', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-broken'; + seedSession(sessionName, sessionStore); + mockCapture.mockRejectedValue(new Error('AX bridge crashed')); + + const data = expectOkData( + await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }), + ); + + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(false); + expect(settle.diff).toBeUndefined(); + expect(settle.hint).toMatch(/Settle observation unavailable/); + // The scroll itself was dispatched and reported success. + expect(mockDispatch.mock.calls.filter(([, command]) => command === 'scroll')).toHaveLength(1); + // Nothing was published, so the scroll's own leaf seam still owns the frame. + expect(sessionStore.get(sessionName)?.refFrameState).toBe('expired'); +}); diff --git a/src/daemon/generic-settle-observation.ts b/src/daemon/generic-settle-observation.ts new file mode 100644 index 000000000..6d8cc7b89 --- /dev/null +++ b/src/daemon/generic-settle-observation.ts @@ -0,0 +1,85 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SettleObservation, SettleParams } from '@agent-device/contracts/interaction'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PostActionObservationCommandName } from '../core/command-descriptor/post-action-observation.ts'; +import type { ContextFromFlags } from './handlers/interaction-common.ts'; +import { readSettleRequest, settleFlagGuardResponse } from './handlers/interaction-flags.ts'; +import { createInteractionRuntime } from './handlers/interaction-runtime.ts'; +import { captureSnapshotForSession } from './handlers/interaction-snapshot.ts'; +import { issueSettleObservationRefs } from './session-snapshot.ts'; +import type { SessionStore } from './session-store.ts'; +import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; + +/** + * `--settle` (#1101) for the generic daemon route (#1638) — today `scroll` and + * `back`, the two generic leaves carrying the descriptor post-action + * observation trait. + * + * It lives in its own module, loaded lazily by request-generic-dispatch.ts only + * when a settle-capable command actually asks to settle, for the same reason + * every daemon ROUTE is a lazy seam (src/daemon/request-handler-chain.ts): + * observing pulls in the whole interaction runtime, and no other generic leaf — + * home, screenshot, orientation, tv-remote — should pay for it in import cost + * or in comprehension (the static edge would fold that runtime cluster into the + * daemon's largest type cycle). + */ + +type GenericSettleReadiness = { settle: SettleParams | undefined } | { response: DaemonResponse }; + +/** + * Reads the settle request off the flags, refusing an orphaned + * `--settle-quiet`. Called before the device action so a grammar mistake costs + * no device work — and only for a command the caller already checked carries + * the post-action observation trait, which is what makes the name cast sound. + */ +export function readGenericSettleRequest( + command: string, + flags: CommandFlags | undefined, +): GenericSettleReadiness { + const invalid = settleFlagGuardResponse(command as PostActionObservationCommandName, flags); + if (invalid) return { response: invalid }; + return { settle: readSettleRequest(flags) }; +} + +/** + * Runs the target-less settle through the interaction runtime and attaches the + * observation to the command's response payload. + * + * Best-effort by contract (the settle engine never throws): the device action + * already succeeded, so a broken observation degrades to a hint rather than + * failing the command. + */ +export async function observeGenericSettle(params: { + req: DaemonRequest; + session: SessionState; + sessionName: string; + logPath: string; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + settle: SettleParams; + /** The session's STORED pre-action tree, read before the action dispatched. */ + baselineNodes: SnapshotNode[]; +}): Promise<{ settle: SettleObservation }> { + const { req, session, sessionName, settle, baselineNodes } = params; + const runtime = createInteractionRuntime({ + req, + sessionName, + logPath: params.logPath, + sessionStore: params.sessionStore, + contextFromFlags: params.contextFromFlags, + captureSnapshotForSession, + }); + const observation = await runtime.interactions.observeSettle({ + ...settle, + session: sessionName, + requestId: req.meta?.requestId, + baselineNodes, + }); + // ADR 0014: a settled diff publishes refs minted from the stored settled + // tree, so the response activates a partial frame and carries the generation + // those refs are pinned to — the same dance the touch route runs. + const refsGeneration = issueSettleObservationRefs(session, observation); + return { + settle: refsGeneration === undefined ? observation : { ...observation, refsGeneration }, + }; +} diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index fa530dd8c..9a91fdf5c 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -38,7 +38,7 @@ import { type DirectIosSelectorTarget, } from '../direct-ios-selector.ts'; import { expireRefFrame } from '../ref-frame.ts'; -import { markSessionPartialRefsIssued, resolveRefStalenessWarning } from '../session-snapshot.ts'; +import { issueSettleObservationRefs, resolveRefStalenessWarning } from '../session-snapshot.ts'; import type { DaemonResponse, SessionState } from '../types.ts'; import { assertAndroidPressStayedInApp, @@ -326,41 +326,10 @@ async function buildTargetedTouchResponsePayloads(params: { referenceFrame, extra, staleRefsWarning: params.staleRefsWarning, - settleRefsGeneration: settleRefsGenerationIssue(session, result), + settleRefsGeneration: issueSettleObservationRefs(session, result.settle), }); } -/** - * #1101 `--settle`: a settle observation carrying a diff hands the client refs - * minted from the freshly stored settled tree (added lines carry them), which - * makes the response ref-issuing like snapshot/find: it activates a PARTIAL - * frame (ADR 0014) authorizing exactly those bodies, and the stored tree's - * generation rides inside the settle payload for MCP per-ref pinning. Without a - * diff — never captured, or sparse-quality capture that was not stored — nothing - * was issued and the frame is left as the press's leaf seam expired it. - */ -function settleRefsGenerationIssue( - session: SessionState, - result: PressCommandResult | FillCommandResult | LongPressCommandResult, -): number | undefined { - if (!result.settle?.diff) return undefined; - // ADR 0014: a settled diff publishes the refs it exposed, so it activates a - // PARTIAL frame authorizing exactly those bodies (not the whole tree). - markSessionPartialRefsIssued(session, collectSettleIssuedRefBodies(result.settle)); - return session.snapshotGeneration; -} - -/** The reusable refs a settled diff exposed: added diff lines, `refs`, `tail`. */ -function collectSettleIssuedRefBodies(settle: NonNullable): string[] { - const bodies: string[] = []; - for (const line of settle.diff?.lines ?? []) { - if (line.ref) bodies.push(line.ref); - } - for (const entry of settle.refs ?? []) bodies.push(entry.ref); - for (const entry of settle.tail ?? []) bodies.push(entry.ref); - return bodies; -} - function readLongPressResultDuration(result: TargetedTouchResult): number | undefined { return 'durationMs' in result ? result.durationMs : undefined; } @@ -787,7 +756,7 @@ function buildFillResponsePayloads(params: { referenceFrame, extra: { text: params.text, ...maestroFallback.extra }, staleRefsWarning: params.staleRefsWarning, - settleRefsGeneration: settleRefsGenerationIssue(session, result), + settleRefsGeneration: issueSettleObservationRefs(session, result.settle), }); } diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 828aad2c9..f53f2f262 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -1,5 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SettleObservation, SettleParams } from '@agent-device/contracts/interaction'; import { dispatchCommand } from '../core/dispatch.ts'; +import { commandSupportsSettleObservation } from '../core/command-descriptor/registry.ts'; import { requireCommandSupported } from './handlers/response.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonCommandContext } from './context.ts'; @@ -51,11 +53,19 @@ export async function dispatchGenericCommand(params: { const readinessResponse = await ensureGenericCommandReady(session, platformCommand); if (readinessResponse) return readinessResponse; + const settleReadiness = await readGenericSettleRequest(platformCommand, req.flags); + if ('response' in settleReadiness) return settleReadiness.response; + const settleRequest = settleReadiness.settle; const preflightReadiness = await ensureNoAndroidBlockingDialogReady(session, platformCommand); if ('response' in preflightReadiness) return preflightReadiness.response; const { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags } = resolveCommandPositionals(req); + // #1638 `--settle` baseline: the STORED pre-action tree, captured before the + // action replaces it. Unlike press — whose baseline is the tree it just + // resolved its target against — this can be several commands old; that is the + // honest reading of the #1101 contract, and the diff says so by construction. + const preActionNodes = session.snapshot?.nodes ?? []; const actionStartedAt = Date.now(); const dispatchContext = { @@ -117,7 +127,42 @@ export async function dispatchGenericCommand(params: { flags: req.flags, }); - return { ok: true, data: data ?? {} }; + // Ordering is load-bearing: settle runs AFTER markDeferredInteractionOutcome + // so its first capture is the one that folds the #1542 post-gesture + // stabilization, and after the Android blocking-dialog postflight so a + // recovered dialog is not what the settled diff describes. + const settleData = settleRequest + ? await observeGenericSettle({ + ...params, + settle: settleRequest, + baselineNodes: preActionNodes, + }) + : undefined; + + return { ok: true, data: { ...(data ?? {}), ...(settleData ?? {}) } }; +} + +/** + * The generic route's `--settle` seam, kept behind a lazy import on purpose: + * only `scroll` and `back` carry the descriptor post-action observation trait, + * and observing pulls in the whole interaction runtime. Every other generic leaf + * answers here without loading it, and reaches the rest of dispatch with no + * settle request. + */ +async function readGenericSettleRequest( + command: string, + flags: CommandFlags | undefined, +): Promise<{ settle: SettleParams | undefined } | { response: DaemonResponse }> { + if (!commandSupportsSettleObservation(command)) return { settle: undefined }; + const settleObservation = await import('./generic-settle-observation.ts'); + return settleObservation.readGenericSettleRequest(command, flags); +} + +async function observeGenericSettle( + params: Parameters<(typeof import('./generic-settle-observation.ts'))['observeGenericSettle']>[0], +): Promise<{ settle: SettleObservation }> { + const settleObservation = await import('./generic-settle-observation.ts'); + return await settleObservation.observeGenericSettle(params); } async function ensureNoAndroidBlockingDialogReady( diff --git a/src/daemon/session-event-action.ts b/src/daemon/session-event-action.ts index 121119cf0..ea6e1be64 100644 --- a/src/daemon/session-event-action.ts +++ b/src/daemon/session-event-action.ts @@ -285,6 +285,13 @@ const COMMON_SAFE_FLAG_SPEC = { ], } as const satisfies SafeFlagSpec; +// The `--settle` (#1101) request shape, shown for every command that accepts it: +// the touch commands, and the generic-route `scroll`/`back` (#1638). +const SETTLE_SAFE_FLAGS = { + booleans: [{ source: 'settle' }], + numbers: [{ source: 'settleQuietMs' }], +} as const satisfies Required>; + const SAFE_ACTION_FLAG_SPECS: Record = { [PUBLIC_COMMANDS.open]: { booleans: [{ source: 'relaunch' }, { source: 'testIme' }], @@ -296,7 +303,8 @@ const SAFE_ACTION_FLAG_SPECS: Record = { [PUBLIC_COMMANDS.fill]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.type]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.scroll]: { - numbers: [{ source: 'pixels' }, { source: 'durationMs' }], + booleans: SETTLE_SAFE_FLAGS.booleans, + numbers: [{ source: 'pixels' }, { source: 'durationMs' }, ...SETTLE_SAFE_FLAGS.numbers], }, [PUBLIC_COMMANDS.gesture]: gestureSafeFlagSpec(), [PUBLIC_COMMANDS.swipe]: gestureSafeFlagSpec(), @@ -322,6 +330,7 @@ const SAFE_ACTION_FLAG_SPECS: Record = { numbers: [{ source: 'snapshotDepth', output: 'depth' }], }, [PUBLIC_COMMANDS.back]: { + ...SETTLE_SAFE_FLAGS, enums: [{ source: 'backMode', output: 'mode', values: BACK_MODES }], }, [PUBLIC_COMMANDS.record]: { @@ -353,13 +362,13 @@ function buildSafeActionFlags(action: SessionAction): Record | function touchSafeFlagSpec(): SafeFlagSpec { return { - booleans: [{ source: 'doubleTap' }, { source: 'settle' }], + booleans: [{ source: 'doubleTap' }, ...SETTLE_SAFE_FLAGS.booleans], numbers: [ { source: 'count' }, { source: 'intervalMs' }, { source: 'holdMs' }, { source: 'jitterPx' }, - { source: 'settleQuietMs' }, + ...SETTLE_SAFE_FLAGS.numbers, ], enums: [{ source: 'clickButton', values: CLICK_BUTTONS }], }; @@ -367,8 +376,8 @@ function touchSafeFlagSpec(): SafeFlagSpec { function textEntrySafeFlagSpec(): SafeFlagSpec { return { - booleans: [{ source: 'settle' }], - numbers: [{ source: 'delayMs' }, { source: 'settleQuietMs' }], + booleans: SETTLE_SAFE_FLAGS.booleans, + numbers: [{ source: 'delayMs' }, ...SETTLE_SAFE_FLAGS.numbers], }; } diff --git a/src/daemon/session-snapshot.ts b/src/daemon/session-snapshot.ts index 8e77f6f31..28c60564b 100644 --- a/src/daemon/session-snapshot.ts +++ b/src/daemon/session-snapshot.ts @@ -1,4 +1,5 @@ import { randomInt } from 'node:crypto'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { activatePartialRefFrame, refFrameEpoch, refFrameState } from './ref-frame.ts'; import type { SessionState } from './types.ts'; @@ -120,6 +121,40 @@ export function markSessionPartialRefsIssued(session: SessionState, refs: Iterab activatePartialRefFrame(session, scope); } +/** + * #1101 `--settle`: a settle observation carrying a diff hands the client refs + * minted from the freshly stored settled tree (added lines carry them), which + * makes the response ref-issuing like snapshot/find: it activates a PARTIAL + * frame (ADR 0014) authorizing exactly those bodies, and returns the stored + * tree's generation to ride inside the settle payload for MCP per-ref pinning. + * Without a diff — never captured, or a sparse-quality capture that was not + * stored — nothing was issued, so the frame is left exactly as the action's + * own side-effect seam expired it. + * + * Every route that can attach a settle observation goes through here (touch + * commands and the generic scroll/back route, #1638), so the issuance rule has + * one implementation rather than one per handler. + */ +export function issueSettleObservationRefs( + session: SessionState, + settle: SettleObservation | undefined, +): number | undefined { + if (!settle?.diff) return undefined; + markSessionPartialRefsIssued(session, collectSettleIssuedRefBodies(settle)); + return session.snapshotGeneration; +} + +/** The reusable refs a settled diff exposed: added diff lines, `refs`, `tail`. */ +function collectSettleIssuedRefBodies(settle: SettleObservation): string[] { + const bodies: string[] = []; + for (const line of settle.diff?.lines ?? []) { + if (line.ref) bodies.push(line.ref); + } + for (const entry of settle.refs ?? []) bodies.push(entry.ref); + for (const entry of settle.tail ?? []) bodies.push(entry.ref); + return bodies; +} + /** * Warning for a ref pinned to a generation (`@e12~s3`) whose epoch no longer * matches the session's current ref-frame epoch (`refFrameEpoch`) — NOT the