diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index a905cf2f5..4eff74e73 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -5,6 +5,7 @@ export { isNodeVisibleInEffectiveViewport, isNodeVisibleOnScreen, isUsefulVisibilityAnchor, + isViewportRootNode, isTapPointInsideViewport, resolveEffectiveViewportRect, resolveViewportRect, diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 35ef996bc..a041e6284 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -1,5 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { defineStringEnum } from './string-enum.ts'; +import { isViewportRootNode } from './snapshot-visibility.ts'; import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; // What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to): @@ -273,7 +274,7 @@ export function parseScrollDirection(direction: string): ScrollDirection { function inferViewportRect(nodes: Array>): Rect | undefined { const candidate = nodes - .filter((node) => isViewportNode(node.type) && isValidRect(node.rect)) + .filter((node) => isViewportRootNode(node) && isValidRect(node.rect)) .map((node) => node.rect) .sort( (left, right) => @@ -290,12 +291,6 @@ function inferViewportRect(nodes: Array>): R return { x: 0, y: 0, width, height }; } -function isViewportNode(type: string | undefined): boolean { - if (!type) return false; - const normalized = type.toLowerCase(); - return normalized.includes('application') || normalized.includes('window'); -} - function isValidRect(rect: Rect | undefined): rect is Rect { return !!rect && rect.width > 0 && rect.height > 0; } diff --git a/packages/contracts/src/snapshot-viewport-root.test.ts b/packages/contracts/src/snapshot-viewport-root.test.ts new file mode 100644 index 000000000..faf382941 --- /dev/null +++ b/packages/contracts/src/snapshot-viewport-root.test.ts @@ -0,0 +1,219 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, test } from 'vitest'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..'); +import { isViewportRootNode } from './snapshot-visibility.ts'; + +/** + * `isViewportRootNode` pinned over the vocabulary each backend ACTUALLY emits + * into `node.type`/`role`/`subrole` — not over fixture strings. + * + * Nine hand-rolled spellings of this predicate existed before #1592's method was + * applied here: three normalizing `type|role|subrole`, five lowercasing or + * normalizing `type` alone, and one comparing the normalized type for EQUALITY. + * Measured over the 75 names in the three tables below they agree on 71. All + * four disagreements are macOS windows, and they split two ways: + * + * - `AXFloatingWindow` / `AXSystemFloatingWindow`: only the `===` spelling + * (maestro) misses them, and its platform union is `android | ios`, so it + * can never see a macOS node. Inert — the same shape #1592 found for its + * own role/subrole arm. + * - `AXSystemDialog` / `AXUnknown`: an `AXWindow` whose subrole is emitted AS + * the type, so the six type-only spellings cannot see they are windows while + * `role` names it exactly. These two are the entire behavioral delta of the + * consolidation, and they are windows by role. + * + * Grounding this in emitted vocabulary is what made the collapse provable; a + * fixture-string comparison would have reported divergences no backend can + * produce (#1592, second commit). + * + * Keep these tables honest against their emitters — a predicate that silently + * stops matching the root is invisible in every other test, because the root + * node is scenery in all of them. The macOS table asserts its own completeness + * against the emitter's fixed-output set below; iOS and Android do not, so a + * new `elementTypeName` case or a new Android container class still has to be + * added here by hand. + */ + +// apple/runner/.../RunnerTests+Snapshot.swift `elementTypeName` — a closed set. +// `Element(N)` is the escape hatch for element types the switch does not name. +const IOS_EMITTED_TYPES = [ + 'Application', + 'Window', + 'Button', + 'Cell', + 'StaticText', + 'TextField', + 'TextView', + 'SecureTextField', + 'Switch', + 'Slider', + 'Link', + 'Image', + 'NavigationBar', + 'TabBar', + 'CollectionView', + 'Table', + 'ScrollView', + 'Toolbar', + 'SearchField', + 'SegmentedControl', + 'Stepper', + 'Picker', + 'ActivityIndicator', + 'ProgressIndicator', + 'CheckBox', + 'MenuItem', + 'WebView', + 'Other', + 'Keyboard', + 'Key', + 'Element(72)', +] as const; + +// src/platforms/android/ui-hierarchy.ts — `type` is the uiautomator `class` +// attribute verbatim, a fully-qualified Java class name. +const ANDROID_EMITTED_TYPES = [ + 'android.view.View', + 'android.view.ViewGroup', + 'android.widget.Button', + 'android.widget.EditText', + 'android.widget.FrameLayout', + 'android.widget.HorizontalScrollView', + 'android.widget.ImageButton', + 'android.widget.ImageView', + 'android.widget.LinearLayout', + 'android.widget.ScrollView', + 'android.widget.SeekBar', + 'android.widget.Switch', + 'android.widget.TextView', + 'androidx.compose.ui.platform.ComposeView', + 'androidx.recyclerview.widget.RecyclerView', + 'android.widget.ListView', + 'android.widget.GridView', + 'androidx.core.widget.NestedScrollView', +] as const; + +// apple/macos-helper/.../SnapshotTraversal.swift `normalizedSnapshotType`. Three +// output classes, all represented below: +// 1. every one of the 13 roles the switch maps to a fixed short name; +// 2. `AXWindow`, whose output is the SUBROLE unless that subrole is +// `AXStandardWindow` — so a window's `type` can be any subrole string; +// 3. the `default:` arm, `subrole ?? role`, which emits the raw `AX`-prefixed +// value for every unmapped role. +// This backend is the only one that populates `role`/`subrole` at all. +const MACOS_EMITTED_NODES = [ + // 1. the complete fixed-output set. + { type: 'Application', role: 'AXApplication' }, + { type: 'Sheet', role: 'AXSheet' }, + { type: 'Dialog', role: 'AXDialog' }, + { type: 'Button', role: 'AXButton' }, + { type: 'StaticText', role: 'AXStaticText' }, + { type: 'TextField', role: 'AXTextField' }, + { type: 'TextArea', role: 'AXTextArea' }, + { type: 'ScrollArea', role: 'AXScrollArea' }, + { type: 'Group', role: 'AXGroup' }, + { type: 'MenuBar', role: 'AXMenuBar' }, + { type: 'MenuBarItem', role: 'AXMenuBarItem' }, + { type: 'Menu', role: 'AXMenu' }, + { type: 'MenuItem', role: 'AXMenuItem' }, + // 2. AXWindow: standard subrole collapses to "Window", every other subrole + // is emitted verbatim as the type. + { type: 'Window', role: 'AXWindow', subrole: 'AXStandardWindow' }, + { type: 'AXFloatingWindow', role: 'AXWindow', subrole: 'AXFloatingWindow' }, + { type: 'AXSystemFloatingWindow', role: 'AXWindow', subrole: 'AXSystemFloatingWindow' }, + { type: 'AXSystemDialog', role: 'AXWindow', subrole: 'AXSystemDialog' }, + { type: 'AXUnknown', role: 'AXWindow', subrole: 'AXUnknown' }, + // 3. the raw `subrole ?? role` fallback for unmapped roles. + { type: 'AXWebArea', role: 'AXWebArea' }, + { type: 'AXList', role: 'AXList' }, + { type: 'AXTable', role: 'AXTable' }, + { type: 'AXOutline', role: 'AXOutline' }, + { type: 'AXScrollBar', role: 'AXScrollBar' }, + { type: 'AXSplitGroup', role: 'AXSplitGroup' }, + { type: 'AXToolbar', role: 'AXToolbar' }, + // A subrole on an UNMAPPED role: the default arm returns `subrole ?? role`, so the + // subrole wins. (`AXTextField` cannot appear here — its own arm always returns + // `TextField`, whatever the subrole.) + { type: 'AXSortButton', role: 'AXCell', subrole: 'AXSortButton' }, +] as const; + +/** + * The emitter's fixed outputs, READ FROM THE EMITTER — not a second hand-kept + * list. A hand-kept twin compared against a hand-kept table is circular: both + * drift together and the comparison stays green (#1613 review). Parsing the + * Swift switch is what makes "a role added there fails here" a real claim. + */ +function macosFixedOutputsFromEmitter(): string[] { + const swift = fs.readFileSync( + path.join( + REPO_ROOT, + 'apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift', + ), + 'utf8', + ); + const fn = /private func normalizedSnapshotType\([\s\S]*?\n\}/.exec(swift)?.[0]; + if (!fn) throw new Error('normalizedSnapshotType not found — the emitter moved; fix this parser'); + // `case "AXWindow":` returns a subrole expression, not a literal, so it is + // deliberately absent from the literal-return set the table pins. + return [...fn.matchAll(/return "([A-Za-z]+)"/g)].map((match) => match[1]!).sort(); +} + +describe('isViewportRootNode over emitted backend vocabulary', () => { + test('iOS: exactly Application and Window, out of 31 emitted names', () => { + const roots = IOS_EMITTED_TYPES.filter((type) => isViewportRootNode({ type })); + expect(roots).toEqual(['Application', 'Window']); + }); + + test('iOS: substring and equality agree, so the collapse of the `===` spelling was a no-op', () => { + for (const type of IOS_EMITTED_TYPES) { + const equality = type === 'Application' || type === 'Window'; + expect({ type, root: isViewportRootNode({ type }) }).toEqual({ type, root: equality }); + } + }); + + // The load-bearing one. Android has no root node, so `resolveViewportRect`'s + // third fallback (largest containing rect of any node) is the only arm that + // ever returns on Android — and the resolvers that lack it return null there. + test('Android: no emitted class name is a viewport root', () => { + const roots = ANDROID_EMITTED_TYPES.filter((type) => isViewportRootNode({ type })); + expect(roots).toEqual([]); + }); + + test('macOS: every AXWindow subrole is a root, whatever `type` says', () => { + const roots = MACOS_EMITTED_NODES.filter(isViewportRootNode).map((node) => node.type); + expect(roots).toEqual([ + 'Application', + 'Window', + 'AXFloatingWindow', + 'AXSystemFloatingWindow', + 'AXSystemDialog', + 'AXUnknown', + ]); + }); + + // The table's own completeness claim, asserted rather than trusted: every + // fixed output the emitter's switch can return appears above, so adding a + // role to that switch without adding it here fails. + test('macOS: the table covers every fixed output the Swift emitter can return', () => { + const covered = new Set(MACOS_EMITTED_NODES.map((node) => node.type)); + const emitted = macosFixedOutputsFromEmitter(); + expect(emitted.length).toBeGreaterThan(10); + expect(emitted.filter((name) => !covered.has(name))).toEqual([]); + }); + + // Why the canonical predicate reads role/subrole and not `type` alone: these + // two are windows that no type-only spelling could see. + test('macOS: role rescues the two window subroles `type` alone cannot name', () => { + const typeOnly = (type: string) => { + const value = type.toLowerCase(); + return value.includes('application') || value.includes('window'); + }; + const rescued = MACOS_EMITTED_NODES.filter( + (node) => isViewportRootNode(node) && !typeOnly(node.type), + ).map((node) => node.type); + expect(rescued).toEqual(['AXSystemDialog', 'AXUnknown']); + }); +}); diff --git a/packages/contracts/src/snapshot-visibility.ts b/packages/contracts/src/snapshot-visibility.ts index 9771bb71f..3e33fce6e 100644 --- a/packages/contracts/src/snapshot-visibility.ts +++ b/packages/contracts/src/snapshot-visibility.ts @@ -15,6 +15,32 @@ type SnapshotVisibilityNode = Pick< 'rect' | 'index' | 'parentIndex' | 'type' | 'role' | 'subrole' >; +/** + * The application/window root: the node a target rect is measured against, and + * the node whose own rect is invariant under any gesture. + * + * One definition for the whole repo. It reads `type`, `role` AND `subrole` + * because the macOS helper is the only backend that populates the latter two, + * and it is the only backend that can emit a window whose `type` does not say + * so — `normalizedSnapshotType` returns the raw subrole for a non-standard + * window, so an `AXWindow` with subrole `AXSystemDialog` or `AXUnknown` reads + * as neither from `type` alone while `role` names it exactly. + * + * Substring, not equality: macOS emits unmapped roles with their `AX` prefix + * intact and subroles like `AXFloatingWindow` that are windows by any reading. + * iOS emits a closed set of 31 short names in which only `Application` and + * `Window` contain either word, so substring and equality agree there. Android + * emits fully-qualified Java class names and no root node at all, so no + * spelling of this predicate matches anything on Android — see + * `resolveViewportRect`'s third fallback, which is what Android actually uses. + */ +export function isViewportRootNode(node: Pick): boolean { + const kind = [node.type, node.role, node.subrole] + .map((value) => normalizeType(value ?? '')) + .join(' '); + return kind.includes('application') || kind.includes('window'); +} + /** * The root viewport a target rect is measured against: the largest * Application/Window rect containing the target's center, falling back to the @@ -22,30 +48,19 @@ type SnapshotVisibilityNode = Pick< */ export function resolveViewportRect(nodes: RawSnapshotNode[], targetRect: Rect): Rect | null { const targetCenter = centerOfRect(targetRect); - const rectNodes = nodes.filter((node) => hasValidRect(node.rect)); - const viewportNodes = rectNodes.filter((node) => { - const type = (node.type ?? '').toLowerCase(); - return type.includes('application') || type.includes('window'); - }); - - const containingViewport = pickLargestRect( - viewportNodes - .map((node) => node.rect as Rect) - .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)), + const rects = nodes.flatMap((node) => + hasValidRect(node.rect) ? [{ node, rect: node.rect }] : [], ); - if (containingViewport) return containingViewport; - - const viewportFallback = pickLargestRect(viewportNodes.map((node) => node.rect as Rect)); - if (viewportFallback) return viewportFallback; + const viewportRects = rects + .filter((entry) => isViewportRootNode(entry.node)) + .map((entry) => entry.rect); + const contains = (rect: Rect) => containsPoint(rect, targetCenter.x, targetCenter.y); - const genericContaining = pickLargestRect( - rectNodes - .map((node) => node.rect as Rect) - .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)), + return ( + pickLargestRect(viewportRects.filter(contains)) ?? + pickLargestRect(viewportRects) ?? + pickLargestRect(rects.map((entry) => entry.rect).filter(contains)) ); - if (genericContaining) return genericContaining; - - return null; } function hasValidRect(rect: Rect | undefined): rect is Rect { diff --git a/packages/maestro/src/internal/runtime-port-geometry.ts b/packages/maestro/src/internal/runtime-port-geometry.ts index d4e9d7031..5add0ad29 100644 --- a/packages/maestro/src/internal/runtime-port-geometry.ts +++ b/packages/maestro/src/internal/runtime-port-geometry.ts @@ -6,7 +6,7 @@ import { pointInsideRect } from './shared.ts'; import { findNearestScrollableAncestor, isScrollableNodeLike, - normalizeType, + isViewportRootNode, } from '@agent-device/contracts/snapshot'; import { MAESTRO_COMPATIBILITY_PRESETS } from './compatibility-policy.ts'; import { resolveNumeric } from './engine-flow.ts'; @@ -108,8 +108,7 @@ function findScrollContainer( function findLargestViewportRect(nodes: SnapshotState['nodes']): Rect | undefined { return nodes .filter((node) => { - const type = normalizeType(node.type ?? ''); - return isPositiveFiniteRect(node.rect) && (type === 'application' || type === 'window'); + return isPositiveFiniteRect(node.rect) && isViewportRootNode(node); }) .sort( (left, right) => diff --git a/packages/maestro/src/internal/snapshot-policy.ts b/packages/maestro/src/internal/snapshot-policy.ts index ea3b3f16c..c04569e1f 100644 --- a/packages/maestro/src/internal/snapshot-policy.ts +++ b/packages/maestro/src/internal/snapshot-policy.ts @@ -3,8 +3,13 @@ import { findNearestScrollableAncestor, findSnapshotAncestor, isUsefulVisibilityAnchor, + isViewportRootNode, } from '@agent-device/contracts/snapshot'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { + containsPoint, + isPositiveFiniteRect, + isRectVisibleInViewport, +} from '@agent-device/kernel/rect'; import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; export function isMaestroNodeVisible( @@ -47,15 +52,12 @@ function isVisibleInEffectiveViewport(node: SnapshotNode, nodes: SnapshotNode[]) const viewport = findNearestScrollableAncestor(node, byIndex, (ancestor) => Boolean(ancestor.rect))?.rect ?? resolveRootViewport(nodes, node.rect); - return viewport ? rectsOverlap(node.rect, viewport) : true; + return viewport ? isRectVisibleInViewport(node.rect, viewport) : true; } function resolveRootViewport(nodes: SnapshotNode[], target: Rect): Rect | null { const viewportRects = nodes - .filter((node) => { - const type = (node.type ?? '').toLowerCase(); - return node.rect && (type.includes('application') || type.includes('window')); - }) + .filter((node) => node.rect && isViewportRootNode(node)) .map((node) => node.rect!) .sort((left, right) => right.width * right.height - left.width * left.height); const centerX = target.x + target.width / 2; @@ -64,14 +66,3 @@ function resolveRootViewport(nodes: SnapshotNode[], target: Rect): Rect | null { viewportRects.find((rect) => containsPoint(rect, centerX, centerY)) ?? viewportRects[0] ?? null ); } - -function rectsOverlap(left: Rect, right: Rect): boolean { - return ( - Math.max(left.x, right.x) <= Math.min(left.x + left.width, right.x + right.width) && - Math.max(left.y, right.y) <= Math.min(left.y + left.height, right.y + right.height) - ); -} - -function containsPoint(rect: Rect, x: number, y: number): boolean { - return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; -} diff --git a/src/core/interaction-targeting.ts b/src/core/interaction-targeting.ts index 29d72aedb..8213cdf76 100644 --- a/src/core/interaction-targeting.ts +++ b/src/core/interaction-targeting.ts @@ -1,7 +1,7 @@ import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; import { centerOfRect } from '@agent-device/kernel/snapshot'; import { containsPoint, pickLargestRect } from '@agent-device/kernel/rect'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { normalizeType, isViewportRootNode } from '@agent-device/contracts/snapshot'; import { findNearestHittableAncestor } from '../snapshot/snapshot-processing.ts'; import { isSnapshotNodeInteractionBlocked } from '../snapshot/snapshot-occlusion.ts'; import { @@ -158,10 +158,7 @@ function isScrollingContainer(node: SnapshotNode): boolean { function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect | null { const targetCenter = centerOfRect(targetRect); const viewportRects = nodes - .filter((node) => { - const type = (node.type ?? '').toLowerCase(); - return type.includes('application') || type.includes('window'); - }) + .filter(isViewportRootNode) .map((node) => normalizeRect(node.rect)) .filter((rect): rect is Rect => rect !== null); if (viewportRects.length === 0) return null; diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 596fa8da0..7ecc96933 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -1,4 +1,5 @@ import { dispatchCommand } from '../../core/dispatch.ts'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { findBestMatchesByLocator, isReadOnlyFindAction, @@ -371,8 +372,7 @@ function isRootInteractionContainer( root: SnapshotState['nodes'][number] | undefined, ): boolean { if (!root?.rect || !node.rect) return false; - const type = node.type?.toLowerCase() ?? ''; - if (!type.includes('application') && !type.includes('window')) return false; + if (!isViewportRootNode(node)) return false; return rectsMatch(node.rect, root.rect); } diff --git a/src/daemon/interaction-outcome-policy.ts b/src/daemon/interaction-outcome-policy.ts index a1f9518e6..29ae0b124 100644 --- a/src/daemon/interaction-outcome-policy.ts +++ b/src/daemon/interaction-outcome-policy.ts @@ -3,7 +3,7 @@ import { isMobilePlatform } from '@agent-device/kernel/device'; import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { collectKeyboardChromeRefs } from '../core/snapshot-chrome.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { contextFromFlags } from './context.ts'; import type { SessionState } from './types.ts'; @@ -390,22 +390,7 @@ function isNonDiscriminatingSurfaceNode( node: SnapshotNode, keyboardChromeRefs: ReadonlySet, ): boolean { - return isViewportRootKind(node) || (node.ref !== undefined && keyboardChromeRefs.has(node.ref)); -} - -/** - * Minimal local equivalent of `isViewportRoot` in - * `src/snapshot/snapshot-occlusion.ts` (source of truth) — that function is - * module-private and keyed off the broader `RawSnapshotNode` shape used by - * occlusion/viewport resolution, so it is reimplemented here rather than - * exported solely for this caller. Same normalized-kind substring test; keep - * the two in lockstep if the underlying AX vocabulary changes. - */ -function isViewportRootKind(node: Pick): boolean { - const normalizedKind = [node.type, node.role, node.subrole] - .map((value) => normalizeType(value ?? '')) - .join(' '); - return normalizedKind.includes('application') || normalizedKind.includes('window'); + return isViewportRootNode(node) || (node.ref !== undefined && keyboardChromeRefs.has(node.ref)); } function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined { diff --git a/src/daemon/screenshot-overlay.ts b/src/daemon/screenshot-overlay.ts index 51ba7f804..7991ea026 100644 --- a/src/daemon/screenshot-overlay.ts +++ b/src/daemon/screenshot-overlay.ts @@ -9,7 +9,7 @@ import { import type { PNG } from '../utils/png.ts'; import { decodePngAsync, encodePngAsync } from '../utils/png-worker-client.ts'; import { analyzeReactNativeOverlay } from '../core/react-native-overlay.ts'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot'; import { findNearestAncestor } from '../snapshot/snapshot-processing.ts'; import { resolveAndroidOverlaySourceRect } from './screenshot-overlay-android.ts'; import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts'; @@ -212,7 +212,7 @@ function isAndroidUnlabeledClickableSource( node: SnapshotNode, ): boolean { if (snapshot.backend !== 'android') return false; - if (!node.hittable || !hasPositiveRect(node.rect) || isViewportLikeNode(node)) return false; + if (!node.hittable || !hasPositiveRect(node.rect) || isViewportRootNode(node)) return false; const normalizedType = normalizeType(node.type ?? ''); if (ANDROID_UNLABELED_CLICKABLE_EXCLUDED_TYPES.some((type) => normalizedType.includes(type))) { return false; @@ -318,7 +318,7 @@ function projectRectToScreenshot( function resolveSnapshotBounds(nodes: SnapshotState['nodes']): Rect | null { let viewport: Rect | null = null; for (const node of nodes) { - if (!isViewportLikeNode(node) || !hasPositiveRect(node.rect)) continue; + if (!isViewportRootNode(node) || !hasPositiveRect(node.rect)) continue; if (!viewport || rectArea(node.rect) > rectArea(viewport)) { viewport = node.rect; } @@ -366,7 +366,7 @@ function hasActionableRole(node: SnapshotNode): boolean { } function isOverlayActionableNode(node: SnapshotNode): boolean { - return hasActionableRole(node) && !isViewportLikeNode(node); + return hasActionableRole(node) && !isViewportRootNode(node); } function isProxyOverlayNode(node: SnapshotNode): boolean { @@ -379,15 +379,8 @@ function isProxyOverlayNode(node: SnapshotNode): boolean { ); } -function isViewportLikeNode(node: Pick): boolean { - const roleText = [node.type, node.role, node.subrole] - .map((value) => normalizeType(value ?? '')) - .join(' '); - return roleText.includes('application') || roleText.includes('window'); -} - function isUsableOverlayTarget(node: SnapshotNode | null): node is SnapshotNode { - return Boolean(node?.rect && hasPositiveRect(node.rect) && !isViewportLikeNode(node)); + return Boolean(node?.rect && hasPositiveRect(node.rect) && !isViewportRootNode(node)); } function isMeaningfulSignal(value: string | undefined): boolean { diff --git a/src/snapshot/snapshot-occlusion.ts b/src/snapshot/snapshot-occlusion.ts index d24878bbf..6f0d150ef 100644 --- a/src/snapshot/snapshot-occlusion.ts +++ b/src/snapshot/snapshot-occlusion.ts @@ -2,7 +2,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; import { centerOfRect } from '@agent-device/kernel/snapshot'; import { areRectsApproximatelyEqual, normalizeRect } from '../utils/rect-center.ts'; import { containsPoint } from '@agent-device/kernel/rect'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { normalizeType, isViewportRootNode } from '@agent-device/contracts/snapshot'; const COVERED_PRESENTATION_HINT = 'covered'; const OVERLAY_KIND_FRAGMENTS = [ @@ -220,7 +220,7 @@ function isOverlayLikeNode( options: SnapshotOcclusionOptions, ): boolean { if (!positiveRect(node.rect)) return false; - if (isViewportRoot(node)) return false; + if (isViewportRootNode(node)) return false; if (isFullViewportChromeContainer(node, byIndex)) return false; // This is a presentation-order heuristic: only known floating UI chrome should cover // later targets. Generic hittable containers can appear later without being visually on top. @@ -241,7 +241,7 @@ function isFullViewportChromeContainer( let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; const visited = new Set(); while (current && !visited.has(current.index)) { - if (isViewportRoot(current)) { + if (isViewportRootNode(current)) { const viewportRect = positiveRect(current.rect); return Boolean(viewportRect && areRectsApproximatelyEqual(rect, viewportRect)); } @@ -295,7 +295,7 @@ function isRenderableAdditionalOverlayNode( return ( options.isAdditionalOverlayNode?.(node) === true && positiveRect(node.rect) !== null && - !isViewportRoot(node) + !isViewportRootNode(node) ); } @@ -321,11 +321,6 @@ function normalizeNodeKind(node: Pick normalizeType(value ?? '')).join(' '); } -function isViewportRoot(node: RawSnapshotNode): boolean { - const normalized = normalizeNodeKind(node); - return normalized.includes('application') || normalized.includes('window'); -} - function areRelatedSnapshotNodes( left: RawSnapshotNode, right: RawSnapshotNode,