feat: support --settle on scroll and back (#1638)

.fallowrc.json

@@ -78,6 +78,11 @@
       "file": "src/sdk/*.ts",
       "exports": ["*"]
     },
+    {
+      "comment": "#1642: its three consumers (src/__tests__/cli-device-status.test.ts, src/daemon/__tests__/device-claims.test.ts, src/daemon/__tests__/device-claim-prune.test.ts) all reach it the only way a Vitest module mock can — `vi.mock(path, async (importOriginal) => (await import('...host-process-mock.ts')).pinOwnProcessStartTime(importOriginal))`. Dependency analysis cannot follow that dynamic import to a consumer, the same limitation the daemon route-handler entry above records.",
+      "file": "src/__tests__/test-utils/host-process-mock.ts",
+      "exports": ["pinOwnProcessStartTime"]
+    },
     {
       "comment": "Tool config default exports, loaded by the tool rather than imported.",
       "file": "{tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}",

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- `scroll` and `back` now accept `--settle` (with `--settle-quiet` and `--timeout`), collapsing scroll-then-observe and back-then-observe into one call (#1638). The response carries the same settled payload the touch commands return — verdict, changed-lines diff with fresh refs on added lines, the unchanged-interactive tail, and `refsGeneration` when the settled tree was stored — and is best-effort: it never fails the action. One difference is deliberate: `scroll`/`back` resolve no element, so the diff baseline is the session's stored pre-action tree ("the last tree you observed") rather than a freshly resolved pre-action capture. Both commands now also preserve the daemon on timeout, like the other settle-capable commands.
 - Security: repository `./agent-device.json` now accepts only project-safe automation defaults. It rejects daemon endpoint/auth/transport/server settings, tenant/run/lease selectors, provider/cloud and Metro connection fields, headers, executable reporter modules, local write destinations, and other operator-controlled values before local module loading or any daemon health/RPC request. Put remote endpoint and token together in protected CI environment variables, user config, an explicit `--config` file, or the existing `connect`/`--remote-config` workflow. Daemon auth tokens no longer travel in serialized command flags.
 - `viewport` is now rejected during capability admission on Apple targets instead of reaching the device and failing inside dispatch. No Apple backend can resize a screen — simulator and device geometry is fixed by the selected device type — so `viewport` on iOS/iPadOS/tvOS/macOS now fails with `UNSUPPORTED_OPERATION`, `viewport is not supported on this device`, and a hint pointing at `--platform web` and at picking a different simulator. `capabilities` no longer advertises `viewport` on Apple targets. Web viewport resizing (`agent-device viewport 1280 900 --platform web`) is unchanged, and Android was already denied.
 - `--save-script` is now accepted only by the commands that declare it — `open`, `close`, and `replay`. A hand-built daemon request (or a `batch` step) that set `saveScript` on any other command, such as `record` or `trace`, used to arm script publication and could write a `.ad` artifact; it is now rejected with `INVALID_ARGS` before the request reaches admission, the device, or any handler. CLI, Node, and MCP usage of `--save-script` on its documented commands is unchanged.

CONTEXT.md

@@ -131,11 +131,16 @@ task touches:
   eligibility gate, and the module owns only these post-action markers — never ADR 0014 ref-frame
   expiry or the ADR 0012/0016 staged protocols. Distinct from the same-response settled
   observation below.
-- Settled observation: opt-in (`--settle`) post-action payload on press/click/fill/longpress — the
-  quiet-window stable loop re-captures until the UI settles, and the response carries the diff vs the
-  pre-action tree (changed lines only, added lines with fresh refs, `refsGeneration` when the settled
-  tree was stored). Best-effort: never fails the action; `settled: false` plus a hint on never-quiet
-  content.
+- Settled observation: opt-in (`--settle`) post-action payload on press/click/fill/longpress and, on
+  the generic route, scroll/back — the quiet-window stable loop re-captures until the UI settles, and
+  the response carries the diff vs the pre-action tree (changed lines only, added lines with fresh
+  refs, `refsGeneration` when the settled tree was stored). Best-effort: never fails the action;
+  `settled: false` plus a hint on never-quiet content. Which commands support it is a descriptor
+  trait (`postActionObservation`), and the CLI flags, MCP fields, timeout envelope, and ref-pinning
+  all derive from it. The two routes differ in ONE way, deliberately: the touch commands diff against
+  the freshly resolved pre-action capture, while scroll/back — which resolve nothing — diff against
+  the session's stored pre-action tree, so their diff reads "settled tree vs the last tree you
+  observed".
 - Resolution disclosure (ADR 0012 decision 2): additive `resolution` field on
   press/click/fill/longpress responses discloses how the acting path resolved its target —
   `runtime`/`unique` or `runtime`/`disambiguated` (with `matchCount`/`winnerDiagnostic`/`tiebreak`/

README.md

@@ -61,6 +61,8 @@ agent-device close
 
 Use refs only from the latest output. Do not assume an earlier `@eN` still identifies the same element. After a command with `--settle`, use the refs in its diff. Take another snapshot only if the diff omits what you need.
 
+`--settle` works the same way on `scroll` and `back`, so scroll-then-observe and back-then-observe are one call too.
+
 Snapshots use the app's accessibility tree. Clear labels, roles, and test IDs make agent runs more reliable. Use screenshots and videos as evidence or when accessibility data is poor. Use refs and selectors for actions and assertions when you can.
 
 ![agent-device demo showing Codex using agent-device to create a new contact in the iOS Contacts app from a simple prompt](./website/docs/public/agent-device-contacts.gif)

packages/contracts/src/client-gesture.ts

@@ -135,9 +135,10 @@ export type RotateGestureOptions = DeviceCommandBaseOptions & {
 
 export type TransformGestureOptions = DeviceCommandBaseOptions & TransformGestureParams;
 
-export type ScrollOptions = DeviceCommandBaseOptions & {
-  direction: ScrollInputDirection;
-  amount?: number;
-  pixels?: number;
-  durationMs?: number;
-};
+export type ScrollOptions = DeviceCommandBaseOptions &
+  SettleCommandOptions & {
+    direction: ScrollInputDirection;
+    amount?: number;
+    pixels?: number;
+    durationMs?: number;
+  };

packages/contracts/src/navigation.ts

@@ -1,5 +1,6 @@
 import type { BackMode } from './back-mode.ts';
 import type { DeviceRotation } from './device-rotation.ts';
+import type { SettleObservation } from './interaction.ts';
 import type { TvRemoteButton } from './tv-remote.ts';
 
 /**
@@ -18,11 +19,17 @@ export type HomeCommandResult = {
   message: string;
 };
 
-/** `back` — `{ action: 'back', mode, message: 'Back' }`; `mode` defaults to `'in-app'`. */
+/**
+ * `back` — `{ action: 'back', mode, message: 'Back' }`; `mode` defaults to
+ * `'in-app'`. The one field the generic route may add on top of the dispatch
+ * handler's literal return: `settle`, the opt-in `--settle` observation
+ * (#1638), attached after the command by the generic dispatcher.
+ */
 export type BackCommandResult = {
   action: 'back';
   mode: BackMode;
   message: string;
+  settle?: SettleObservation;
 };
 
 /** `orientation` — `{ action: 'orientation', orientation, message: 'Rotated to <orientation>' }`. */

scripts/__tests__/help-conformance-sample-producers.ts

@@ -5,6 +5,7 @@ import {
   BROWSERSTACK_CONNECT_SAMPLE,
   DEVICE_IN_USE_SAMPLE,
   NOT_SETTLED_SAMPLE,
+  OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
   PRIVATE_AX_RECOVERY_SAMPLE,
   SETTLE_DIFF_SAMPLE,
   SETTLE_DIFF_SAMPLE_NOTES,
@@ -208,6 +209,64 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [
       }).trimEnd();
     },
   },
+  {
+    name: 'OFFSCREEN_TARGET_SNAPSHOT_SAMPLE',
+    producer: 'the visible-first snapshot renderer with off-screen rows summarized',
+    sample: OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
+    render: () => {
+      // A settings-style scrollable list in an 800pt viewport: five rows fit,
+      // four more (Privacy & Security, Notifications, Wallpaper, Developer) are
+      // laid out below it, so visible-first presentation summarizes them and
+      // none of their refs reach the output.
+      const row = (index: number, ref: string, label: string, y: number) => ({
+        index,
+        ref,
+        parentIndex: 2,
+        type: 'Cell',
+        label,
+        interactive: true,
+        hittable: y < 800,
+        rect: { x: 0, y, width: 390, height: 120 },
+      });
+      const nodes = [
+        {
+          index: 0,
+          ref: 'e1',
+          type: 'Application',
+          label: 'Preferences',
+          rect: { x: 0, y: 0, width: 390, height: 800 },
+        },
+        {
+          index: 1,
+          ref: 'e2',
+          parentIndex: 0,
+          type: 'Window',
+          rect: { x: 0, y: 0, width: 390, height: 800 },
+        },
+        {
+          index: 2,
+          ref: 'e3',
+          parentIndex: 1,
+          type: 'CollectionView',
+          interactive: true,
+          rect: { x: 0, y: 60, width: 390, height: 740 },
+        },
+        row(3, 'e4', 'General', 60),
+        row(4, 'e5', 'Display', 190),
+        row(5, 'e6', 'Sounds', 320),
+        row(6, 'e7', 'Focus', 450),
+        row(7, 'e8', 'Screen Time', 580),
+        row(8, 'e9', 'Privacy & Security', 900),
+        row(9, 'e10', 'Notifications', 1030),
+        row(10, 'e11', 'Wallpaper', 1160),
+        row(11, 'e12', 'Developer', 1290),
+      ];
+      return formatSnapshotText(
+        { nodes, backend: 'xctest', truncated: false },
+        { interactiveOnly: true },
+      ).trimEnd();
+    },
+  },
   {
     name: 'DEVICE_IN_USE_SAMPLE',
     producer: 'the real session-open by-session conflict producer',

scripts/help-conformance-cases.mjs

@@ -4,6 +4,7 @@ import {
   BROWSERSTACK_CONNECT_SAMPLE,
   DEVICE_IN_USE_SAMPLE,
   NOT_SETTLED_SAMPLE,
+  OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
   SETTLE_DIFF_SAMPLE,
   SETTLE_DIFF_SAMPLE_NOTES,
   SETTLE_TAIL_SAMPLE,
@@ -377,6 +378,46 @@ Use the output already shown to determine whether the feed-search UI is present,
       { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET },
     ],
   },
+  {
+    // #1638/#1650: the closed --settle grammar grew scroll and back, and this
+    // extension IS the feature's payoff — collapsing scroll-then-observe into
+    // one call. The old guidance framed settle as a mutation suffix, and
+    // scroll reads as navigation, so eligibility generalizing is exactly what
+    // this case checks. The task deliberately does not mention settle: the
+    // wanted row is off-screen with no ref anywhere in the output, the
+    // tempting pre-#1638 plan is `scroll` + a separate `snapshot -i`, and
+    // acceptance is the single settled call.
+    id: 'sample-output-offscreen-target-scrolls-settled',
+    docs: ['--help:first30'],
+    task: quiz(
+      OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
+      'The task is to open the Notifications row of this list. What command should run next?',
+    ),
+    expectations: ['validPlanCommands', 'fullPrefix'],
+    matchers: [
+      {
+        id: 'scrollsDownSettled',
+        pattern: /(?:^|\n)(?:agent-device\s+)?scroll\s+down\b[^\n]*--settle\b/i,
+      },
+    ],
+    forbidden: [
+      // The two-call habit this case exists to catch: a scroll line without
+      // --settle means a separate observation call is coming.
+      {
+        id: 'noUnsettledScroll',
+        pattern: /(?:^|\n)(?:agent-device\s+)?scroll\b(?:(?!--settle)[^\n])*(?=\n|$)/i,
+      },
+      { id: 'noSnapshot', pattern: /\bsnapshot\b/i },
+      { id: 'noWaitStable', pattern: /wait\s+stable/i },
+      // Notifications never appears in the output, so any bare @eN press is a
+      // guessed ref, not a resolved target.
+      {
+        id: 'noGuessedRef',
+        pattern: /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d/i,
+      },
+      { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET },
+    ],
+  },
   {
     id: 'sample-output-not-settled-needs-observe',
     docs: ['--help:first30'],

scripts/help-conformance-sample-outputs.mjs

@@ -44,6 +44,24 @@ settled after 480ms: +2 -0 (~11 unchanged)
 + @e22 [text] "3 items"`,
 };
 
+// Visible-first snapshot of a scrollable list whose remaining rows sit below
+// the viewport: the off-screen content is summarized, not listed as refs. The
+// scroll-to-find quiz case hangs off this — the wanted row exists but no ref
+// for it appears anywhere in the output.
+export const OFFSCREEN_TARGET_SNAPSHOT_SAMPLE = {
+  command: 'agent-device snapshot -i',
+  output: `Snapshot: 8 visible nodes (12 total)
+@e1 [application] "Preferences"
+@e2 [window]
+@e3 [collection]
+@e4 [cell] "General"
+@e5 [cell] "Display"
+@e6 [cell] "Sounds"
+@e7 [cell] "Focus"
+@e8 [cell] "Screen Time"
+  [content below collection hidden]`,
+};
+
 // Never-settled press: success response, no diff, NEVER_SETTLED_HINT attached.
 export const NOT_SETTLED_SAMPLE = {
   command: 'agent-device press @e12 --settle',

src/cli/parser/__tests__/cli-help-topics.test.ts

@@ -91,7 +91,7 @@ test('usage includes agent workflows, config, environment, and examples footers'
   assert.match(usageText, /Default app loop: agent-device open <app>/);
   assert.match(
     usageText,
-    /Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close/,
+    /Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close/,
   );
   assert.match(usageText, /type never accepts --settle/);
   assert.match(usageText, /explicit success confirmation is visible, stop/);

src/cli/parser/cli-help.ts

@@ -79,7 +79,7 @@ const AGENT_START_LINES = [
   // Haiku from 0/2 baseline to 4/4; generic structured-hint recovery passed 8/8
   // uncoached output cases versus 7/8 with the longer special-case prose.
   'Default app loop: agent-device open <app> -> agent-device snapshot -i -> mutate a current target with --settle -> continue from that settled diff -> agent-device close.',
-  'Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.',
+  'Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.',
   'Follow structured command hints before choosing a recovery action.',
   'Targets are concrete refs or selectors: @e12, label="Query", role=button label="Submit".',
   'Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.',

src/commands/interaction/index.ts

@@ -19,17 +19,9 @@ import type {
   TypeTextOptions,
 } from '@agent-device/contracts/client';
 import type { CommandSchemaOverride } from '../../cli-schema/types.ts';
-import {
-  commandSupportsSettleObservation,
-  commandSupportsVerifyEvidence,
-} from '../../core/command-descriptor/registry.ts';
-import {
-  REPEATED_TOUCH_FLAGS,
-  SELECTOR_SNAPSHOT_FLAGS,
-  SETTLE_FLAGS,
-} from '../cli-grammar/flag-groups.ts';
-import { type FlagKey } from '../cli-grammar/flag-types.ts';
+import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS } from '../cli-grammar/flag-groups.ts';
 import { defineExecutableCommand } from '../command-contract.ts';
+import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts';
 import {
   commonToClientOptions,
   toClientElementTarget,
@@ -137,20 +129,15 @@ const interactionCliSchemas = {
     ],
   },
   scroll: {
-    usageOverride: 'scroll <direction|top|bottom> [amount] [--pixels <n>] [--duration-ms <ms>]',
+    usageOverride:
+      'scroll <direction|top|bottom> [amount] [--pixels <n>] [--duration-ms <ms>] [--settle]',
     positionalArgs: ['directionOrEdge', 'amount?'],
-    allowedFlags: ['pixels', 'durationMs'],
+    allowedFlags: ['pixels', 'durationMs', ...postActionObservationCliFlags('scroll')],
   },
 } as const satisfies Record<string, CommandSchemaOverride>;
 
 type InteractionCommandMetadata = (typeof interactionCommandMetadata)[number];
 type InteractionCommandName = InteractionCommandMetadata['name'];
-function postActionObservationCliFlags(command: InteractionCommandName): readonly FlagKey[] {
-  const flags: FlagKey[] = [];
-  if (commandSupportsVerifyEvidence(command)) flags.push('verify');
-  if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS);
-  return flags;
-}
 
 const clickCommandDefinition = defineExecutableCommand(metadata('click'), (client, input) =>
   client.interactions.click(toClickOptions(input)),
@@ -318,6 +305,7 @@ const scrollCommandFacet = defineCommandFacet({
   cliSchema: interactionCliSchemas.scroll,
   cliReader: interactionCliReaders.scroll,
   daemonWriter: interactionDaemonWriters.scroll,
+  cliOutputFormatter: interactionCliOutputFormatters.scroll,
 });
 
 const getCommandFacet = defineCommandFacet({

src/commands/interaction/interactions.ts

@@ -97,6 +97,7 @@ export const interactionCliReaders = {
   },
   scroll: (positionals, flags) => ({
     ...commonInputFromFlags(flags),
+    ...settleInputFromFlags(flags),
     direction: readScrollDirection(positionals[0]),
     amount: optionalCliNumber(positionals[1]),
     pixels: flags.pixels,