From dbb4c300f8ac1c99d251a6934a373ccd68857cec Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 11 May 2026 13:36:21 +0800 Subject: [PATCH 1/3] {"schema":"decodex/commit/1","summary":"Rework native scroll capture flow and chrome","authority":"manual"} --- Makefile.toml | 8 + README.md | 27 +- docs/decisions/index.md | 2 + docs/decisions/scroll-capture-architecture.md | 140 +++ docs/index.md | 6 + docs/policy.md | 8 +- .../smoke-perf-validation-surface.md | 9 +- .../scroll-capture-prior-art-2026-05-10.json | 232 ++++ docs/runbook/index.md | 2 + docs/runbook/performance-validation.md | 26 +- docs/runbook/scroll-capture-benchmarks.md | 7 +- docs/runbook/scroll-capture-recovery-plan.md | 138 +++ docs/runbook/validate-release.md | 6 +- docs/spec/capture-session.md | 24 +- docs/spec/settings.md | 8 +- native/macos-host/Package.swift | 1 + .../Sources/RsnapHostBridge/HostFFI.swift | 104 ++ .../Sources/RsnapHostBridgeProbe/main.swift | 6 +- .../RsnapNativeHostKit/CaptureHostView.swift | 149 ++- .../CaptureOverlayController.swift | 100 +- ...eSessionController+FrozenInteraction.swift | 83 +- .../CaptureSessionController+Live.swift | 4 + ...ptureSessionController+ScrollCapture.swift | 1002 +++++++++++++++-- .../CaptureSessionController.swift | 20 +- .../FrozenCaptureModels.swift | 19 +- .../LiveChromeWindows.swift | 11 + .../RsnapNativeHostKit/LiveFrameStream.swift | 25 + .../NativeHostSettingsView.swift | 42 +- .../RsnapNativeHostKitProbe/main.swift | 28 + packages/rsnap-capture-core/src/session.rs | 25 +- .../rsnap-host-ffi/include/rsnap_host_ffi.h | 22 +- packages/rsnap-host-ffi/src/lib.rs | 191 +++- .../src/host_live_sampling_macos.rs | 49 + packages/rsnap-overlay/src/lib.rs | 24 + .../src/overlay/tests/worker_tick_runtime.rs | 65 +- packages/rsnap-overlay/src/scroll_capture.rs | 175 ++- .../src/scroll_capture/downward_resolution.rs | 32 +- .../src/scroll_capture/support.rs | 66 +- .../rsnap-overlay/src/scroll_capture/tests.rs | 131 ++- scripts/smoke/lib/live-hud.sh | 168 ++- .../lib/native-visual-contract-summary.py | 70 ++ scripts/smoke/lib/pasteboard-image-info.swift | 19 + .../smoke/lib/scroll-background-command.swift | 24 + .../smoke/lib/scroll-background-window.swift | 184 +++ scripts/smoke/lib/scroll-wheel-burst.swift | 59 + scripts/smoke/native-scroll-capture-macos.sh | 480 ++++++++ scripts/smoke/native-visual-contract-macos.sh | 16 +- 47 files changed, 3797 insertions(+), 240 deletions(-) create mode 100644 docs/decisions/scroll-capture-architecture.md create mode 100644 docs/research/scroll-capture-prior-art-2026-05-10.json create mode 100644 docs/runbook/scroll-capture-recovery-plan.md create mode 100644 scripts/smoke/lib/pasteboard-image-info.swift create mode 100644 scripts/smoke/lib/scroll-background-command.swift create mode 100644 scripts/smoke/lib/scroll-background-window.swift create mode 100644 scripts/smoke/lib/scroll-wheel-burst.swift create mode 100755 scripts/smoke/native-scroll-capture-macos.sh diff --git a/Makefile.toml b/Makefile.toml index 99582a13..5a0f0ebc 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -152,6 +152,10 @@ args = [ "native/macos-host/Sources", "scripts/smoke/lib/live-hud-mouse-path.swift", "scripts/smoke/lib/mask-probe-capture.swift", + "scripts/smoke/lib/pasteboard-image-info.swift", + "scripts/smoke/lib/scroll-background-window.swift", + "scripts/smoke/lib/scroll-background-command.swift", + "scripts/smoke/lib/scroll-wheel-burst.swift", "scripts/smoke/lib/visual-background-window.swift", ] @@ -167,6 +171,10 @@ args = [ "native/macos-host/Sources", "scripts/smoke/lib/live-hud-mouse-path.swift", "scripts/smoke/lib/mask-probe-capture.swift", + "scripts/smoke/lib/pasteboard-image-info.swift", + "scripts/smoke/lib/scroll-background-window.swift", + "scripts/smoke/lib/scroll-background-command.swift", + "scripts/smoke/lib/scroll-wheel-burst.swift", "scripts/smoke/lib/visual-background-window.swift", ] diff --git a/README.md b/README.md index 3fd0f3c9..559975c5 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ https://github.com/user-attachments/assets/ff2fe84f-f551-40e8-919c-66ae8a61f8e7 - In Frozen mode, `Space` copies the current frozen PNG to the clipboard and exits. - In Frozen mode, Cmd+S (macOS) / Ctrl+S saves the current PNG to disk and exits. - On macOS, Frozen mode can recognize text from the current capture and copy the result to the clipboard from the toolbar. -- Frozen toolbar tools include pointer, pen, arrow, text, mosaic, spotlight, undo, redo, auto-center, - OCR, copy, and save. +- Frozen toolbar tools include pointer, pen, arrow, text, mosaic, spotlight, undo, redo, + auto-center, OCR, Scroll Capture for dragged-region freezes, copy, and save. - `Esc` cancels capture. - Glass HUD with Classic Glass fallback and Liquid Glass in release builds on supported macOS. - Tab-triggered loupe sample and frozen-mode toolbar for quick action access. @@ -74,7 +74,10 @@ Prototype / in active development. - Menubar and Dock are not included in live window-outline targeting. - Windows support is planned (minimum Windows 10), but not implemented yet. - The scroll-capture engine, deterministic replay, and benchmark surfaces remain in the repository, - but the v0.2.1 native-host release does not expose scroll capture in the toolbar. + but the v0.2.1 native-host release does not expose scroll capture in the toolbar. On this + development branch, scroll capture uses ordered ScreenCaptureKit region frames, overlay-local + wheel forwarding, and Rust-owned fail-closed stitching on macOS. Release readiness for broader + target apps is governed by `docs/runbook/scroll-capture-recovery-plan.md`. ## Usage @@ -122,13 +125,15 @@ After Gatekeeper allows the app to open, continue with Screen Recording permissi ### macOS permissions -Rsnap currently relies on **Screen Recording** permission to capture other apps/windows. +Rsnap requires **Screen Recording** permission to capture other apps/windows. - ScreenCaptureKit live sampling on macOS requires macOS 12.3+ and Screen Recording permission. - Normal region/window/monitor capture does not require Accessibility or Input Monitoring. -- The retained scroll-capture path uses Screen Recording-backed screenshots plus forwarded wheel - input, but the v0.2.1 native-host release does not expose scroll capture in the toolbar. +- The retained scroll-capture path uses Screen Recording-backed screenshots plus overlay-local + wheel forwarding; it does not require Accessibility, Input Monitoring, Accessibility target + acquisition, app scripting, or browser/DOM access. The v0.2.1 native-host release does not expose + scroll capture in the toolbar. - macOS may describe Screen Recording as `Screen & System Audio Recording` or as direct screen/audio access when Rsnap bypasses the system picker. -- Settings -> Permissions shows Screen Recording as the only required permission. +- Settings -> Permissions shows Screen Recording as the required capture permission. - Normal native capture depends on Screen Recording; if access is missing, Rsnap opens the Screen Recording page in System Settings and shows a floating drag-to-grant guide. - You can reopen the Permissions section from `Settings…` in the tray or menubar menu at any time. - Base capture path: `System Settings` -> `Privacy & Security` -> `Screen Recording`. @@ -163,7 +168,13 @@ Rsnap currently relies on **Screen Recording** permission to capture other apps/ Scroll capture is temporarily hidden in the v0.2.1 native-host release. The retained Rust scroll-capture session, deterministic replay, and benchmark surfaces remain for validation and -future re-enablement, but users should not expect a `Scroll Capture` toolbar item in this release. +future re-enablement, but users should not expect a `Scroll Capture` toolbar item in that release. + +On this development branch, scroll capture targets dragged-region Frozen capture on macOS. The +implementation commits downward growth only after ordered-frame pairwise registration plus overlap +proof, fails closed on weak registration or rewind, and forwards wheel input to target apps through +one universal path. Follow `docs/runbook/scroll-capture-recovery-plan.md` for release-scope +validation. ## Development diff --git a/docs/decisions/index.md b/docs/decisions/index.md index deba8c3a..283ab445 100644 --- a/docs/decisions/index.md +++ b/docs/decisions/index.md @@ -49,3 +49,5 @@ Then keep the body decision-oriented: screenshot annotation over faithful pointer-path reproduction - `docs/decisions/frozen-toolbar-anchor.md` for the stable-anchor layout choice that prevents style-capsule expansion from moving the primary Frozen toolbar +- `docs/decisions/scroll-capture-architecture.md` for the accepted layered scroll-capture target + architecture based on CleanShot/Xnip/Snagit/Shottr/ScrollSnap prior art and Rsnap live failures diff --git a/docs/decisions/scroll-capture-architecture.md b/docs/decisions/scroll-capture-architecture.md new file mode 100644 index 00000000..e7859298 --- /dev/null +++ b/docs/decisions/scroll-capture-architecture.md @@ -0,0 +1,140 @@ +# Scroll Capture Architecture + +Status: accepted and implemented for the current macOS validation path + +Date: 2026-05-10 + +Context: Earlier scroll-capture attempts passed deterministic tests but failed live use: tearing, +sparse appends after the page had already reached the bottom, false joins after rewind, and a first +frozen toolbar frame dominated by tint. The product target is closer to CleanShot/Xnip behavior: +start from any dragged region over a scrollable app, let scrolling feel native, append only proven +content, and fail closed instead of creating a bad stitched image. + +Decision: Rsnap uses one generic product path for macOS Scroll Capture: the focused overlay receives +wheel input inside the selected viewport, forwards that input to the underlying target with the +original wheel magnitude through short all-overlay passthrough windows, samples ordered +ScreenCaptureKit region frames only around input bursts, falls back to a below-overlay region capture +when the live stream does not provide a fresh region, and Rust owns monotonic registration/commit. +Accessibility is not part of the product path; Rsnap does not use Accessibility target acquisition, +settable AX scroll bars, app scripting, browser/DOM access, or a cancellable CGEvent tap. + +Consequences: Scroll Capture is no longer limited to apps exposing settable AX scroll bars. It needs +Screen Recording because frames come from ScreenCaptureKit. Rsnap observes wheel input only as an +input, forwarding, and sampling signal. Wheel deltas are not treated as content movement authority, +because trackpad/mouse deltas do not reliably map to viewport pixels. + +Supporting research run: `docs/research/scroll-capture-prior-art-2026-05-10.json`. + +## Prior-Art Findings + +| Source | What matters | Rsnap decision | +| --- | --- | --- | +| ScrollSnap | Open-source Swift uses ScreenCaptureKit region screenshots, overlay mouse passthrough, a repeating capture timer, and Vision translational registration. | Copy the macOS shape: region capture, temporary overlay passthrough, and Vision as an offset proposal. Do not copy its correctness model: it advances the previous frame after failed registration and crops committed output on upward motion. | +| wayscrollshot | Captures the selected region continuously while the user scrolls, skips duplicate signatures, and appends only the new bottom slice after overlap proof. | Copy the universal loop: user scrolls naturally; capture runs continuously; stitching is fail-closed and append-only. | +| ShareX | Uses repeated rectangle captures, configurable scroll methods, duplicate-image stopping, best-match overlap, and bottom-edge ignore for sticky chrome. | Copy the idea that scroll capture is a loop with explicit stop/failure status and overlap search that ignores unstable edges. Do not make a platform-specific message/scrollbar path the only product path. | +| Xnip public guide | Documents same-portion matching and pausing when matching fails; warns about fast, dynamic, nonvertical, and upward scrolls. | Bad input must pause/no-op, not guess. Downward growth is the only committed direction. | +| CleanShot URL API | Exposes scrolling capture as a first-class mode, including auto-scroll parameters. | Product entry should be direct and obvious, but proprietary internals are not evidence for a required AX design. | + +## Architecture + +### Entry + +Scroll Capture starts only from an editable dragged-region frozen capture. The toolbar button and +plain `s` both call the same native entry point. On start: + +- freeze the selected region and create the Rust scroll stitch session from that exact first frame; +- switch the frozen toolbar to non-editing scroll-capture state; +- forward overlay-local wheel events through short all-overlay passthrough windows; +- keep a global scroll-wheel observer only as diagnostics/fallback telemetry; +- start the ordered ScreenCaptureKit region-frame sampling loop. + +### Sampling + +The capture loop drains ordered live region frames by sequence number. It does not debounce to a +single latest frame and does not sample from a stale cache as commit authority. When the live stream +has no ordered frame for the selected region, Swift captures the same region below the overlay and +still sends it through the Rust overlap gate instead of appending blindly. Sampling is bounded to +short windows after scroll input instead of running forever on the main actor, so toolbar clicks and +cancel remain responsive while intermediate repaint states are still observed and rejected or +committed in order. + +Wheel events are not movement authority. Overlay-local wheel handling treats each event as an input +signal, forwards the real wheel magnitude to the target while all overlay windows temporarily ignore +mouse events, and samples the resulting ScreenCaptureKit frames. Reverse/upward input may move the +underlying viewport, but it cannot mutate or crop the committed canvas. The marker on synthetic events +prevents feedback loops. + +### Registration And Commit + +Rust owns all commit decisions: + +- Vision pairwise registration can propose downward motion between adjacent ordered frames. +- Pixel overlap corroboration must confirm the proposal before append. +- Growth is monotonic and downward-only. +- A blocked overshot frame does not become the committed frontier. +- Upward motion records rewind/observation state but never crops or mutates committed output. +- After rewind, growth resumes only after the previous committed frontier is reacquired and the + viewport advances beyond it. +- Ambiguous overlap, low-information content, sticky/changing bands, large skipped gaps, and dynamic + repaint states become no-commit states. + +Copy/save always exports the same committed canvas shown in the minimap preview. + +### Permissions + +Required: + +- Screen Recording, because ScreenCaptureKit supplies the capture frames. + +Not required for Scroll Capture: + +- Accessibility or Input Monitoring; +- AX target acquisition or settable scroll bars; +- target app scripting or browser/DOM access. + +## Rejected Options + +| Option | Why rejected | +| --- | --- | +| Latest-frame passive sampling | It can drop intermediate frames and append only a tiny tail after the page already reached the bottom. | +| AX-controlled product path | It only works for targets exposing controllable accessibility scroll bars, which violates the goal of one generic mode across arbitrary apps. | +| Permanent overlay passthrough | It makes the toolbar unreachable and loses control of the capture UI. Rsnap instead uses short all-overlay passthrough windows only while forwarding one wheel event. | +| ScrollSnap-style timer as commit clock | A timer is useful as a sampling mechanism, but correctness cannot advance the comparison anchor after failed registration. | +| Wheel delta as motion authority | Wheel/trackpad delta is input intent, not content motion. It can differ by device, target app, acceleration, rubber-banding, and scroll position. | +| Browser/DOM full-page capture | Useful as a future specialized browser path, but it does not cover arbitrary macOS apps/windows. | + +## Implementation Status + +- The native host starts Scroll Capture in `manual_universal` mode and emits + `capture.scroll_capture_mode outcome=manual_universal`. +- The native host emits `capture.scroll_input_tap outcome=not_used` and uses overlay-local wheel + forwarding as the release-quality input path. +- Synthetic wheel forwarding preserves the real wheel magnitude for target app feel; wheel magnitude + is never used as pixel motion authority, and reverse/upward viewport motion is observed without + append. +- Rust's worker-pairwise path handles ordered frames, overlap corroboration, overshot blocking, and + rewind/reacquire behavior. +- Settings exposes Screen Recording as the only required permission for Scroll Capture. +- The first frozen toolbar frame is covered by the native visual contract tint check. +- Active scroll-capture toolbar glass keeps the configured Settings tint/material but leaves the + toolbar backing in live HUD-style transparency so Liquid Glass samples the real live content + instead of a frozen/tinted surrogate. + +## Validation Contract + +Before calling Scroll Capture fixed: + +- `scripts/smoke/native-scroll-capture-macos.sh` must pass with `SCROLL_DRIVER=wheel` for keyboard + start and toolbar start. +- `scripts/smoke/native-visual-contract-macos.sh` must pass and keep toolbar tint dominance below + its threshold. +- Rust scroll-capture tests must pass, including overshot and rewind/reacquire cases. +- `cargo make checks` must pass before handoff. + +## Sources + +- ScrollSnap: https://github.com/Brkgng/ScrollSnap +- wayscrollshot: https://github.com/jswysnemc/wayscrollshot +- ShareX scrolling capture: https://github.com/ShareX/ShareX +- Xnip scrolling capture guide: https://www.xnipapp.com/scrolling-capture/ +- CleanShot URL API: https://cleanshot.com/docs-api diff --git a/docs/index.md b/docs/index.md index bf4f5ec5..2f5a88b6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,8 @@ The active split below is by question type, not by human-versus-agent audience. `docs/runbook/` - Need native-host or Rust telemetry collection and summary steps -> `docs/runbook/telemetry-debugging.md` +- Need to recover macOS scroll capture after live tearing, sparse stitching, or rollback failures + -> `docs/runbook/scroll-capture-recovery-plan.md` - Need the step-by-step execution sequence for a host/core reset slice -> `docs/runbook/architecture-reset-implementation.md` - Need the active architecture-reset target and migration posture -> @@ -47,6 +49,10 @@ The active split below is by question type, not by human-versus-agent audience. `docs/reference/workspace-layout.md` - Need durable rationale for the architecture reset -> `docs/decisions/native-host-rust-core-reset.md` +- Need the accepted layered scroll-capture architecture and prior-art analysis -> + `docs/decisions/scroll-capture-architecture.md` +- Need the supporting machine-readable research run for scroll-capture prior-art analysis -> + `docs/research/scroll-capture-prior-art-2026-05-10.json` - Need generic repo gate names -> `Makefile.toml` - Need smoke or perf validation entrypoints -> `scripts/smoke/` and `scripts/perf/` - Need documentation placement or authoring rules -> `docs/policy.md` diff --git a/docs/policy.md b/docs/policy.md index f5f7586d..81a04312 100644 --- a/docs/policy.md +++ b/docs/policy.md @@ -4,8 +4,8 @@ Purpose: Define how agent-facing documentation is organized, updated, and kept c across this repository. Audience: All documentation under `docs/` is written for AI agents and LLM workflows. -The active split between `spec`, `runbook`, `reference`, and `decisions` is by task shape, not by -reader type. +The active split between `spec`, `runbook`, `reference`, `decisions`, and supporting `research` +artifacts is by task shape, not by reader type. ## Principles @@ -25,6 +25,7 @@ reader type. | Runbook | `docs/runbook/` | Which sequence should I execute? | Runbooks, migrations, validation, troubleshooting | Any procedure or operational change | | Reference | `docs/reference/` | How is it currently organized or implemented? | Ownership maps, implementation-model notes, non-normative technical context | Any layout, ownership, or current-implementation explanation change | | Decisions | `docs/decisions/` | Why was this tradeoff accepted? | Durable rationale for accepted technical or product choices | Any accepted decision with long-lived consequences | +| Research | `docs/research/` | What evidence supported a bounded investigation? | Machine-readable supporting research runs, not primary behavior authority | Any evidence-backed research run that must remain replayable | ## Placement rules @@ -34,6 +35,9 @@ reader type. defining correctness, it belongs in `docs/reference/`. - If a document records why a durable tradeoff was accepted, which alternatives were considered, and what consequences follow from that choice, it belongs in `docs/decisions/`. +- If a document records a bounded research method, evidence inventory, challenge, or decision + finalization, it belongs in `docs/research/` and must link to the authoritative spec, runbook, + reference, or decision it supports. - If a document becomes historical-only and no longer helps execute current work, remove it from `docs/` instead of keeping it in the active routing surface. - Do not duplicate the same authoritative content across documents. Link to the source diff --git a/docs/reference/smoke-perf-validation-surface.md b/docs/reference/smoke-perf-validation-surface.md index 9420ae8e..32a3290f 100644 --- a/docs/reference/smoke-perf-validation-surface.md +++ b/docs/reference/smoke-perf-validation-surface.md @@ -17,9 +17,11 @@ Depends on: `docs/runbook/performance-validation.md`; `docs/spec/performance.md` Covers: The current layer map for smoke/perf entrypoints, deterministic replay/bench surfaces, overlay runtime integration tests, and scroll-capture session semantics tests. -Release exposure note: v0.2.1 hides user-facing scroll capture in the native host. The -scroll-capture entries in this reference describe retained internal validation assets, not a -visible toolbar feature in that release. +Release exposure note: v0.2.1 hides user-facing scroll capture in the native host. On this +development branch, scroll capture is implemented behind the current validation contract. The +scroll-capture entries in this reference describe retained validation assets and recovery surfaces; +follow `docs/runbook/scroll-capture-recovery-plan.md` before making a release-scope readiness claim +for broader target apps. ## Layer definitions @@ -39,6 +41,7 @@ visible toolbar feature in that release. | `scripts/smoke/analyze-scroll-capture-trace.sh` | Script entrypoint | deterministic replay | Emits summary-only replay analysis for semantic drift triage. | | `scripts/smoke/native-hud-follow-macos.sh` | Script entrypoint | live macOS perf smoke | HUD/loupe follow-cadence smoke for performance work, including delivered mouse-event count, sample refresh cadence, active-layer chrome cadence, and frame-tick cadence. | | `scripts/smoke/native-visual-contract-macos.sh` | Script entrypoint | live macOS smoke | Core native-host behavior contract: repeated real click freezes, repeated held drag freezes, in-drag and frozen screenshots, click/drag editability, border-leak, scrim, and handoff telemetry gates. | +| `scripts/smoke/native-scroll-capture-macos.sh` | Script entrypoint | live macOS scroll smoke | Real frozen-region scroll-capture smoke on a deterministic scrollable native window; asserts the session is unlocked, ScreenCaptureKit exposes a display, drag freeze occurs, Scroll Capture starts in `manual_universal` mode, overlay-local wheel forwarding is the input path, real wheel input moves the target through short all-overlay passthrough windows, selected-region frames sample through the live stream or below-overlay fallback, no legacy auto-scroll telemetry appears, and multiple committed growth events append before copy/export. The default driver is `SCROLL_DRIVER=wheel`; `SCROLL_DRIVER=notification` is retained for direct background-control diagnosis. | | `scripts/smoke/self-check-macos.sh` | Script entrypoint | smoke readiness | Verifies native HUD-follow smoke tooling readiness without the real GUI run. | | `scripts/smoke/macos.sh` | Script entrypoint | smoke aggregation | Runs the core native visual contract and HUD-follow responsiveness smoke. | | `scripts/perf/local.sh` | Script entrypoint | deterministic benches | Runs the committed Criterion smoke-sized benchmark sweep. | diff --git a/docs/research/scroll-capture-prior-art-2026-05-10.json b/docs/research/scroll-capture-prior-art-2026-05-10.json new file mode 100644 index 00000000..cdaec030 --- /dev/null +++ b/docs/research/scroll-capture-prior-art-2026-05-10.json @@ -0,0 +1,232 @@ +{ + "schema": "research-run/2", + "run_id": "scroll-capture-prior-art-2026-05-10", + "question": "Which scroll screenshot architecture should Rsnap implement after comparing open-source scroll capture projects and mature product behavior?", + "success_criteria": [ + "Separate source-code evidence from proprietary product behavior.", + "Choose an architecture that can work across arbitrary macOS apps without false joins, sparse appends, or rewind corruption.", + "Record the permission consequence so implementation and Settings do not drift." + ], + "constraints": [ + "Do not infer proprietary internals from CleanShot X or Xnip beyond public behavior and docs.", + "Prefer open-source code and local runtime evidence for implementation details.", + "Keep Rsnap's Rust stitcher as the durable confidence gate; Swift remains macOS integration glue." + ], + "stop_rule": "Stop once the evidence can justify one concrete product path and its failure boundaries.", + "primary_hypothesis": "A universal scroll screenshot path should forward overlay-local wheel input to the target app through short passthrough windows, sample selected-region frames around input bursts with a live-stream-first path plus below-overlay fallback, and commit only Rust-proven monotonic downward growth.", + "rival_hypotheses": [ + "An AX-controlled product path is necessary for smooth capture.", + "A ScrollSnap-style timer plus Vision registration is enough by itself.", + "Wheel deltas can be used as movement authority.", + "A browser/DOM-specific capture path is the only reliable approach." + ], + "falsifiers": [ + "If open-source implementations require app-specific Accessibility control to work across apps, the universal manual path is weaker.", + "If Rsnap's Rust pairwise path cannot reject overshot and rewind cases, user-driven capture is unsafe.", + "If overlay-local wheel forwarding cannot move the target app during short passthrough windows, the chosen Swift input path is not viable." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "stop", + "attempt": 1, + "max_attempts": 1, + "session_id": "019e0d6e-5a55-7e72-a666-5dd687d422d2" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 4, + "independent_option_questions": [ + "Can user-driven capture be the one product path instead of AX control?", + "Which parts of ScrollSnap, wayscrollshot, and ShareX should Rsnap copy or reject?", + "What state must the Rust stitcher preserve to avoid sparse appends and rewind false joins?" + ], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "ScrollSnap uses ScreenCaptureKit region capture, overlay mouse passthrough, a 0.25s repeating capture timer, and Vision translational registration against the previous screenshot.", + "source_family": "open_source_code", + "source_locator": "https://github.com/Brkgng/ScrollSnap" + }, + { + "id": "E2", + "kind": "observation", + "summary": "ScrollSnap advances previousImage even when offset detection fails and crops the running image on upward offsets, which can lose the committed frontier or mutate committed output during rewind.", + "source_family": "open_source_code", + "source_locator": "ScrollSnap/Managers/StitchingManager.swift" + }, + { + "id": "E3", + "kind": "observation", + "summary": "wayscrollshot captures the selected region continuously every 45ms while the user scrolls, skips duplicate signatures, and appends only the new bottom slice after overlap matching.", + "source_family": "open_source_code", + "source_locator": "https://github.com/jswysnemc/wayscrollshot" + }, + { + "id": "E4", + "kind": "observation", + "summary": "wayscrollshot's stitcher treats failed confidence as NoMatch, insufficient positive offset as NoProgress, and only appends when overlap proof produces enough new height.", + "source_family": "open_source_code", + "source_locator": "wayscrollshot/src/stitch.rs" + }, + { + "id": "E5", + "kind": "observation", + "summary": "ShareX's scrolling capture loop repeatedly captures a selected rectangle, detects duplicate images for stopping, searches overlap against the accumulated result, ignores unstable bottom edge rows, and reports success or partial success.", + "source_family": "open_source_code", + "source_locator": "https://github.com/ShareX/ShareX" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Xnip documents same-portion matching and pause/failure causes including fast scroll, dynamic content, nonvertical scroll, and upward scroll.", + "source_family": "product_docs", + "source_locator": "https://www.xnipapp.com/scrolling-capture/" + }, + { + "id": "E7", + "kind": "observation", + "summary": "CleanShot exposes scrolling capture as a first-class URL action with start and autoscroll parameters, but the public API does not prove a specific internal stitching implementation.", + "source_family": "product_docs", + "source_locator": "https://cleanshot.com/docs-api" + }, + { + "id": "E8", + "kind": "observation", + "summary": "Rsnap's previous AX-gated path made Scroll Capture usable only in targets exposing controllable accessibility scroll bars, which violated the goal of one generic mode across arbitrary apps.", + "source_family": "repo_runtime", + "source_locator": "manual user acceptance feedback, 2026-05-10" + }, + { + "id": "E9", + "kind": "observation", + "summary": "Rsnap's Rust worker-pairwise tests cover successive growth, overshot blocked frames that must not append tails, and rewind/reacquire before growth resumes.", + "source_family": "repo_tests", + "source_locator": "packages/rsnap-overlay/src/scroll_capture/tests.rs" + }, + { + "id": "E10", + "kind": "observation", + "summary": "Live wheel smoke showed permanent overlay passthrough can make the toolbar unreachable and can let the target jump beyond capturable overlap; the current product path uses short all-overlay passthrough windows for each forwarded wheel event instead.", + "source_family": "repo_runtime", + "source_locator": "target/telemetry/native-host-20260511-002448/native-host.oslog" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "AX control can automate scrolling when target apps expose usable accessibility state, but it cannot be the one generic product path because many apps do not expose settable scroll bars.", + "supporting_evidence_ids": [ + "E7", + "E8" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "User-driven wheel input is the broadest macOS app strategy, but release-quality capture must keep toolbar control responsive while sampling ordered frames around real forwarded input.", + "supporting_evidence_ids": [ + "E1", + "E3", + "E4", + "E5", + "E6", + "E9", + "E10" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Vision or template matching may propose offsets, but committed output must stay monotonic and append-only; failed registration must not advance the committed comparison frontier.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E4", + "E9" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T4", + "summary": "Wheel deltas are useful as input pacing and sampling signals but unsafe as movement authority because device acceleration, app behavior, rubber-banding, and scroll position can change the resulting content motion.", + "supporting_evidence_ids": [ + "E3", + "E4", + "E6", + "E9", + "E10" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "universal_overlay_wheel_ordered_sck_rust_fail_closed", + "rejected_options": [ + "ax_controlled_product_path", + "latest_frame_debounce", + "scrollsnap_timer_without_committed_frontier", + "pure_overlay_passthrough", + "wheel_delta_motion_authority", + "browser_dom_only" + ], + "decision_claim": "Rsnap should implement one universal product path: overlay-local wheel forwarding through short passthrough windows, selected-region frame sampling around input bursts with a live-stream-first path plus below-overlay fallback, and Rust worker-pairwise fail-closed registration. Accessibility is not required and must not be used for target acquisition, settable AX scroll bars, or target-app-specific automation.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6", + "E8", + "E9", + "E10" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3", + "T4" + ] + }, + "judgment_hash": "sha256:32be7c91e2b543d6cc56bea72bd6f09688373ba370bb2ba3eaee447b584e61f2" + }, + { + "seq": 5, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:32be7c91e2b543d6cc56bea72bd6f09688373ba370bb2ba3eaee447b584e61f2", + "summary": "Skeptic found the universal path decision-ready only if wheel events remain non-authoritative, the input path forwards real target scrolling without blocking toolbar control, and docs do not reintroduce AX target control.", + "objections": [] + }, + { + "seq": 6, + "type": "finalized_decision_ready", + "confidence": "medium", + "judgment_hash": "sha256:32be7c91e2b543d6cc56bea72bd6f09688373ba370bb2ba3eaee447b584e61f2", + "missing_evidence": [ + "Live keyboard-start and toolbar-start smokes must pass with real wheel input after implementation.", + "Representative app coverage beyond the deterministic native scroll smoke remains a product acceptance task." + ] + } + ] +} diff --git a/docs/runbook/index.md b/docs/runbook/index.md index ca15a388..dbfe78f6 100644 --- a/docs/runbook/index.md +++ b/docs/runbook/index.md @@ -64,6 +64,8 @@ Then structure the body for execution: - `docs/runbook/architecture-reset-validation.md` for validating host/core reset work without routing through superseded shell-specific assumptions - `docs/runbook/performance-validation.md` for repo-native performance and smoke command routing +- `docs/runbook/scroll-capture-recovery-plan.md` for recovering macOS scroll capture without + treating deterministic checks as product readiness - `docs/runbook/scroll-capture-benchmarks.md` for deterministic scroll-capture benchmark usage - `docs/runbook/telemetry-debugging.md` for collecting and summarizing native-host OSLog plus Rust rolling logs during runtime debugging diff --git a/docs/runbook/performance-validation.md b/docs/runbook/performance-validation.md index 12c421a4..cc96b53a 100644 --- a/docs/runbook/performance-validation.md +++ b/docs/runbook/performance-validation.md @@ -17,10 +17,11 @@ Depends on: `docs/spec/performance.md` Outputs: A clear command choice for the regression class you are testing, plus a repeatable local baseline workflow for the committed Criterion benchmark targets. -Current release status: v0.2.1 hides user-facing scroll capture in the native host. The replay and -benchmark commands in this runbook still own retained internal scroll-capture engine validation and -future re-enablement work, but they are not evidence that the v0.2.1 toolbar exposes scroll -capture. +Current release status: v0.2.1 hides user-facing scroll capture in the native host. On this +development branch, scroll capture is implemented under the validation contract. The replay and +benchmark commands in this runbook still own retained internal scroll-capture engine validation, but +they do not prove that a release toolbar exposes scroll capture and do not replace the recovery plan +or a final target-app acceptance run for broader release claims. ## Command selection @@ -39,6 +40,8 @@ Use the smallest command that matches the regression surface: - Native-host click/drag selection, border leakage, mask stability, visual/material, or live-to-frozen handoff regressions: `scripts/smoke/native-visual-contract-macos.sh` +- Native-host scroll capture start/sample/commit regressions on a deterministic scroll source: + `scripts/smoke/native-scroll-capture-macos.sh` - General local deterministic performance sweep before or after a change: `scripts/perf/local.sh` - Dedicated macOS environment validation without driving the real smoke scenario: @@ -74,13 +77,18 @@ worker-pairwise self-check path when no recorded user trace is available. - Runs the core native visual contract smoke. - Runs the native HUD-follow responsiveness smoke. -For a future downward scroll-capture re-enablement, the expected verification sequence is: +For the current downward scroll-capture path, the expected verification sequence is: 1. `cargo make checks` -2. `scripts/smoke/replay-scroll-capture.sh` -3. `scripts/smoke/analyze-scroll-capture-trace.sh` +2. `scripts/smoke/replay-scroll-capture-self-check.sh` +3. `scripts/smoke/replay-scroll-capture.sh` and + `scripts/smoke/analyze-scroll-capture-trace.sh` when a recorded trace is present 4. any targeted deterministic `cargo test -p rsnap-overlay ...` -5. one fresh release live touchpad run with a newly recorded trace +5. `scripts/smoke/native-scroll-capture-macos.sh` with the default `SCROLL_DRIVER=wheel`; use + `SCROLL_DRIVER=notification` only when directly diagnosing background movement independent of + HID wheel delivery +6. for release-scope claims, one fresh release live touchpad run on a target webpage/app, ideally + with a newly recorded trace For the current ownership map of those scripts versus replay, runtime, and session tests, read `docs/reference/smoke-perf-validation-surface.md`. @@ -111,6 +119,8 @@ Dedicated macOS smoke: - Requires a logged-in macOS desktop session. - Requires the expected Screen Recording and automation permissions for the smoke scripts. +- Fails before driving the app when the macOS session is locked or ScreenCaptureKit returns no + shareable display. Treat that as environment readiness, not scroll-capture correctness evidence. - Covers the native-host HUD-follow desktop path. The hard follow gate uses active pointer-movement cadence (`live_chrome.active_layer_chrome_render_gap`) and frame-tick cadence (`live_chrome.frame_tick_gap`) rather than startup, Tab-expand, or close transition gaps. diff --git a/docs/runbook/scroll-capture-benchmarks.md b/docs/runbook/scroll-capture-benchmarks.md index ea446012..84f105b1 100644 --- a/docs/runbook/scroll-capture-benchmarks.md +++ b/docs/runbook/scroll-capture-benchmarks.md @@ -14,9 +14,10 @@ Depends on: `docs/spec/performance.md` Outputs: A repeatable local benchmark run, an optional saved Criterion baseline, and a clear understanding of what the synthetic fixture is intended to cover. -Current release status: v0.2.1 hides user-facing scroll capture in the native host. This runbook -still applies to the retained internal scroll-capture engine, replay, and future re-enablement -work. +Current release status: v0.2.1 hides user-facing scroll capture in the native host. On this +development branch, scroll capture is implemented under the validation contract. This runbook +applies to the retained internal scroll-capture engine, replay, and stitching validation work; it is +not release-readiness evidence by itself. If you are debugging correctness rather than hot-path speed, route through `docs/runbook/performance-validation.md` first. That runbook owns replay, diff --git a/docs/runbook/scroll-capture-recovery-plan.md b/docs/runbook/scroll-capture-recovery-plan.md new file mode 100644 index 00000000..34d7fe4c --- /dev/null +++ b/docs/runbook/scroll-capture-recovery-plan.md @@ -0,0 +1,138 @@ +# Scroll-Capture Recovery Plan + +Goal: Keep macOS Scroll Capture product-quality without repeating the old patch loop where +deterministic checks passed but live use still tore, lagged, or stitched only a small final region. + +Read this when: scroll capture feels wrong in the running native host, a change touches scroll +sampling, stitching, rewind, toolbar entry, permission copy, or release/readiness claims. + +Inputs: `docs/spec/capture-session.md`; `docs/decisions/scroll-capture-architecture.md`; +`docs/reference/smoke-perf-validation-surface.md`; current user-visible live failures. + +Outputs: A procedural recovery path, explicit acceptance gates, and stop conditions for the current +universal user-scroll architecture. + +## Current Status + +Scroll Capture on this branch uses the accepted universal macOS path: + +- dragged-region Frozen capture as the base frame; +- overlay-local wheel forwarding through short all-overlay passthrough windows; +- ordered ScreenCaptureKit region frames with a below-overlay region fallback when the live stream is + not producing fresh selected-region frames; +- Rust worker-pairwise Vision proposal plus overlap corroboration; +- append-only monotonic downward commits; +- rewind/no-overlap fail-closed behavior; +- first-frozen-toolbar tint protection and active scroll toolbar live-HUD glass backing. + +It does not use Accessibility to acquire or drive target scroll bars, and it does not use wheel +deltas as content movement authority. The remaining readiness bar is live validation: the +deterministic smoke must prove real overlay-local wheel input is forwarded to the target, the target +scrolls during short passthrough windows, pairwise registration sees ordered live frames, and multiple +committed growth events appear. + +The original failure symptoms this plan guards against are: + +- visible tearing in the stitched output; +- preview not following the user's scroll; +- reaching the bottom while only a small amount of content is appended; +- false append after upward rewind; +- first frozen toolbar rendering as an almost fully tint-colored toolbar. + +## Prior-Art Anchors + +Authoritative prior-art analysis lives in `docs/decisions/scroll-capture-architecture.md` and the +machine-readable run at `docs/research/scroll-capture-prior-art-2026-05-10.json`. + +Procedural takeaways: + +- ScrollSnap shows the right macOS shell: ScreenCaptureKit region capture, temporary overlay + passthrough, and Vision registration. +- wayscrollshot shows the right generic loop: continuous selected-region capture while the user + scrolls, duplicate skipping, and append only after overlap proof. +- ShareX shows the right workflow shape: selected rectangle, repeated captures, explicit status, + duplicate stopping, and unstable-edge handling. +- Xnip's public guide confirms that match failure, fast scroll, dynamic content, upward scroll, and + nonvertical movement must pause/no-op instead of producing a guessed stitch. + +## Non-Negotiable Rules + +- Never append rows from weak, ambiguous, stale, or large-gap registration. +- Never treat wheel delta as viewport movement authority. +- Never let raw fast wheel bursts pass directly to the target during release-quality Scroll Capture. +- Never repost a global wheel event that the target app already received. +- Never let a blocked overshot frame become the new committed frontier. +- Never crop or mutate committed output on upward rewind. +- Growth resumes after rewind only when the last committed viewport is reacquired and the viewport + advances beyond it. +- Preview, copy, and save must render the same committed canvas. +- A ready claim requires a fresh unlocked-desktop native scroll smoke. + +## Recovery Checklist + +1. Confirm product entry. + - Drag-region Frozen capture must expose the toolbar Scroll Capture button. + - Plain `s` and the toolbar button must both emit `capture.scroll_capture_entry` with the correct + source and then `capture.scroll_capture_started`. + - `capture.scroll_capture_mode` must report `outcome=manual_universal`. + +2. Confirm input delivery. + - During Scroll Capture, `capture.scroll_input_tap` must report `outcome=not_used`. + - Real wheel input inside the selected viewport must emit `capture.scroll_wheel_intercepted` with + `source=overlay`. + - The global wheel observer may emit `capture.scroll_wheel_observed` as diagnostic telemetry. + - It must not produce legacy `capture.scroll_auto_*` telemetry. + - The deterministic smoke background must report `offsetY > 0` for `SCROLL_DRIVER=wheel`. + +3. Confirm ordered sampling. + - Samples must come from `ordered_live_stream_region`. + - The smoke must show `capture.scroll_sample_observed`. + - Missing live stream frames may show a waiting status, but a passing smoke must eventually sample + and commit. + +4. Confirm stitching authority. + - Multiple `outcome=committed` samples must appear before copy/save. + - Export height growth must exceed the smoke threshold. + - Unsupported direction and no-overlap states must leave export height unchanged. + - Rust tests for overshot blocked frames and rewind/reacquire must remain green. + +5. Confirm visual polish. + - The native visual contract must pass. + - Toolbar tint dominance must stay below the configured threshold on the first frozen frame. + - Active scroll toolbar Liquid Glass must preserve Settings tint/material semantics; do not lower + tint to hide the issue. + +## Validation Order + +Run these in order for this branch: + +1. `scripts/smoke/native-scroll-capture-macos.sh` with default `SCROLL_DRIVER=wheel` and + `SCROLL_START_METHOD=keyboard`. +2. `scripts/smoke/native-scroll-capture-macos.sh` with `SCROLL_DRIVER=wheel` and + `SCROLL_START_METHOD=toolbar`. +3. `scripts/smoke/native-visual-contract-macos.sh`. +4. `cargo make test-host-reset`. +5. `cargo make checks`. + +For broader release readiness, add at least one manual release-build acceptance pass on a long +webpage or representative scrollable app: + +- slow scroll: continuous, non-torn growth; +- medium scroll: tracks or pauses without tearing; +- fast flick: no bad append; +- upward rewind: no append while moving upward; +- resume after rewind: appends only after reacquiring the committed viewport; +- bottom: final canvas includes the observed path instead of only the final viewport. + +## Stop Conditions + +Stop and revisit the architecture instead of stacking another patch when any of these are true: + +- Forwarded wheel input does not reach the underlying target during short passthrough windows. +- ScreenCaptureKit cannot provide coherent ordered frames for the selected region. +- Ordinary user scroll regularly jumps beyond overlap before Rsnap can sample intermediate frames. +- Rust registration cannot distinguish real growth from repeated or dynamic content without + accepting false joins. + +In those cases the honest outcome is a narrower supported contract or a separate specialized mode, +not a best-guess stitcher. diff --git a/docs/runbook/validate-release.md b/docs/runbook/validate-release.md index 3b60f7ca..f76a4b0f 100644 --- a/docs/runbook/validate-release.md +++ b/docs/runbook/validate-release.md @@ -58,8 +58,12 @@ Validate these user-visible flows: fullscreen fallback. - Frozen toolbar tools: pointer, pen, arrow, text, mosaic, spotlight, undo, redo, auto-center, Recognize Text, copy, and save. -- Scroll capture is hidden in the v0.2.1 native-host release: the toolbar must not show a scroll +- For the v0.2.1 native-host release, scroll capture is hidden: the toolbar must not show a scroll capture item, and pressing `s` must not enter scroll capture. +- If a later release candidate includes Scroll Capture, `docs/runbook/scroll-capture-recovery-plan.md` + must have exited successfully first. Then Scroll Capture must stay absent for window-click and + fullscreen freezes, must start from a dragged-region freeze via toolbar or plain `s`, and must + pass the live acceptance run in that plan. - Light and dark appearance; Classic Glass and Liquid Glass where the OS and current build support Liquid Glass. - Settings -> About update rows: `Auto Update` and `Release Version` must use Title Case for row diff --git a/docs/spec/capture-session.md b/docs/spec/capture-session.md index e31df9db..03045116 100644 --- a/docs/spec/capture-session.md +++ b/docs/spec/capture-session.md @@ -17,7 +17,7 @@ Defines: - capture-session entry, live-mode, frozen-mode, and export invariants - the display-first Frozen contract - the distinction between display readiness and export readiness -- the current scroll-capture exposure gate and internal contract when the feature is enabled +- the scroll-capture exposure gate and internal stitching contract - the presence of Frozen-mode annotation state in session output - the existence of a separate Frozen-toolbar layout contract for primary-toolbar anchoring @@ -162,15 +162,25 @@ product level rather than binding itself to a particular window toolkit or shell ## Scroll capture +This section defines the target contract. The current development branch must follow +`docs/runbook/scroll-capture-recovery-plan.md` before claiming that the implementation satisfies +this contract. + - The v0.2.1 native-host release does not expose scroll capture. The frozen toolbar MUST NOT show a scroll-capture item while the native-host scroll-capture gate is disabled, and plain `s` MUST NOT enter scroll capture in that state. -- When scroll capture is re-enabled, it is available only from a dragged-region freeze on macOS. -- When scroll capture is re-enabled, the frozen toolbar may expose `Scroll Capture Down`, and plain - `s` may start scroll capture, whenever the frozen capture source is a dragged region on macOS. -- Scroll capture uses discrete monitor-region screenshots from the native platform capture API as - the source of truth for committed downward growth. -- Pairwise image registration plus overlap proof between adjacent discrete screenshots is the +- If scroll capture is exposed, it is available only from a dragged-region freeze on macOS. +- The frozen toolbar may expose `Scroll Capture`, and plain `s` may start scroll capture, only when + the frozen capture source is a dragged region on macOS. +- Scroll capture uses ordered monitor-region frames from the native platform capture API as the + source of truth for committed downward growth. +- Product scroll capture must not depend on Accessibility target acquisition, settable AX scroll + bars, or target-app-specific automation. It starts one generic wheel-input path after the + dragged-region frozen frame becomes the stitch base. +- While scroll capture is active, Rsnap should forward overlay-local wheel events inside the selected + viewport through short all-overlay passthrough windows. Wheel deltas are input and forwarding + signals only; they must not be treated as content-movement authority. +- Pairwise image registration plus overlap proof between adjacent ordered frames is the source of truth for downward scroll progress, viewport reacquisition, and append eligibility. - Stitching is downward-only: - downward motion may append committed rows diff --git a/docs/spec/settings.md b/docs/spec/settings.md index 993e835f..10c81910 100644 --- a/docs/spec/settings.md +++ b/docs/spec/settings.md @@ -114,14 +114,14 @@ Defines: - Settings must include a Permissions section. - Screen Recording permission is required for the current native macOS capture host. -- Settings must present Screen Recording as the only permission needed by the current native macOS - capture host. +- Settings must present Screen Recording as required for native capture and Scroll Capture. +- Settings must not present Accessibility or Input Monitoring as required for Scroll Capture; the + current product path uses overlay-local wheel forwarding rather than Accessibility target control + or a CGEvent tap. - The Open at Login control must live at the bottom of the Permissions section so OS-owned app access controls remain first. - When Screen Recording is missing at launch or at capture start, Rsnap must open the macOS Screen Recording privacy page and present a small Rsnap-owned floating drag guide near System Settings. -- Accessibility and Input Monitoring must not be displayed in Settings while the current native host - does not need them. - Permission recovery should provide a visible drag-the-app affordance for adding Rsnap to System Settings where macOS allows that workflow, including a directional guide from the floating window toward System Settings and an Open System Settings fallback. diff --git a/native/macos-host/Package.swift b/native/macos-host/Package.swift index e5815389..533b6792 100644 --- a/native/macos-host/Package.swift +++ b/native/macos-host/Package.swift @@ -60,6 +60,7 @@ let package = Package( ], linkerSettings: [ .linkedFramework("AppKit"), + .linkedFramework("ApplicationServices"), .linkedFramework("Carbon"), .linkedFramework("CoreMedia"), .linkedFramework("CoreVideo"), diff --git a/native/macos-host/Sources/RsnapHostBridge/HostFFI.swift b/native/macos-host/Sources/RsnapHostBridge/HostFFI.swift index 125ee2c5..d2cff6f3 100644 --- a/native/macos-host/Sources/RsnapHostBridge/HostFFI.swift +++ b/native/macos-host/Sources/RsnapHostBridge/HostFFI.swift @@ -70,6 +70,22 @@ public struct RGBARegionSnapshot: Equatable, Sendable { } } +public struct RGBARegionFrameSnapshot: Equatable, Sendable { + public var frameSequence: UInt64 + public var frameAgeMicroseconds: UInt64 + public var region: RGBARegionSnapshot + + public init( + frameSequence: UInt64, + frameAgeMicroseconds: UInt64, + region: RGBARegionSnapshot + ) { + self.frameSequence = frameSequence + self.frameAgeMicroseconds = frameAgeMicroseconds + self.region = region + } +} + private func rsnapOwnedRgbaSnapshot(from outRegion: RsnapOwnedRgbaRegion) -> RGBARegionSnapshot? { @@ -2164,6 +2180,36 @@ public final class RsnapScrollCaptureSession: @unchecked Sendable { return try decode(result: outResult) } + public func observeDownwardFrame( + _ frame: RGBARegionSnapshot, + motionRowsHint: Int?, + allowBurstSearch: Bool = true + ) throws -> ScrollObserveResult { + stateLock.lock() + defer { stateLock.unlock() } + + var outResult = RsnapScrollObserveResult() + let hint = UInt32(max(motionRowsHint ?? 0, 0)) + let status = frame.rgba.withUnsafeBytes { buffer -> RsnapStatus in + guard let baseAddress = buffer.bindMemory(to: UInt8.self).baseAddress else { + return RSNAP_STATUS_INVALID_INPUT + } + return rsnap_scroll_session_observe_downward_frame_with_motion_hint( + handle, + UInt32(max(frame.width, 0)), + UInt32(max(frame.height, 0)), + baseAddress, + frame.rgba.count, + hint, + allowBurstSearch ? 1 : 0, + &outResult + ) + } + try rsnapRequireOk(status, context: "observing scroll-capture frame with motion hint") + + return try decode(result: outResult) + } + public func exportImage() throws -> RGBARegionSnapshot? { stateLock.lock() defer { stateLock.unlock() } @@ -2366,6 +2412,64 @@ public final class RsnapLiveSampler: @unchecked Sendable { return rsnapOwnedRgbaSnapshot(from: ownedRegion) } + public func nextRegionFrame( + monitor: MonitorSnapshot, + rect: CGRect, + afterFrameSequence: UInt64, + waitForFresh: Bool + ) throws -> RGBARegionFrameSnapshot? { + stateLock.lock() + defer { stateLock.unlock() } + + let encodedMonitor = RsnapMonitorRect( + id: monitor.id, + origin: RsnapPoint( + x: Int32(monitor.frame.origin.x.rounded()), + y: Int32(monitor.frame.origin.y.rounded()) + ), + width: UInt32(max(monitor.frame.width.rounded(), 0)), + height: UInt32(max(monitor.frame.height.rounded(), 0)), + scale_factor_x1000: monitor.scaleFactorX1000 + ) + let encodedRect = RsnapRect( + x: Int32(rect.origin.x.rounded()), + y: Int32(rect.origin.y.rounded()), + width: UInt32(max(rect.width.rounded(), 0)), + height: UInt32(max(rect.height.rounded(), 0)) + ) + var frameSequence: UInt64 = 0 + var frameAgeMicroseconds: UInt64 = 0 + var ownedRegion = RsnapOwnedRgbaRegion() + let status = rsnap_live_sampler_take_next_region_rgba_after_seq( + handle, + encodedMonitor, + encodedRect, + afterFrameSequence, + UInt8(waitForFresh ? 1 : 0), + &frameSequence, + &frameAgeMicroseconds, + &ownedRegion + ) + let code = rsnap_status_code(status) + if code == 3 { + return nil + } + if code != 0 { + throw HostBridgeError.ffiStatus( + context: "taking next live RGBA region frame", + code: code + ) + } + guard let region = rsnapOwnedRgbaSnapshot(from: ownedRegion) else { + return nil + } + return RGBARegionFrameSnapshot( + frameSequence: frameSequence, + frameAgeMicroseconds: frameAgeMicroseconds, + region: region + ) + } + /// Returns the live sampler's cache-only full-monitor snapshot. /// /// This API does not expose the original frame capture time or stream sequence. Do not use it diff --git a/native/macos-host/Sources/RsnapHostBridgeProbe/main.swift b/native/macos-host/Sources/RsnapHostBridgeProbe/main.swift index cc51f00e..1257b938 100644 --- a/native/macos-host/Sources/RsnapHostBridgeProbe/main.swift +++ b/native/macos-host/Sources/RsnapHostBridgeProbe/main.swift @@ -125,7 +125,7 @@ enum RsnapHostBridgeProbe { scene.statusMessage == nil, scene.toolbarItems.contains(where: { $0.kind == .pointer && $0.selected }), scene.toolbarItems.contains(where: { $0.kind == .ocr && $0.enabled }), - scene.toolbarItems.contains(where: { $0.kind == .scroll }) == false, + scene.toolbarItems.contains(where: { $0.kind == .scroll && $0.enabled }), scene.toolbarItems.contains(where: { $0.kind == .copy && $0.enabled }), scene.toolbarItems.contains(where: { $0.kind == .save && $0.enabled }) else { @@ -135,8 +135,8 @@ enum RsnapHostBridgeProbe { private static func verifyFrozenToolbarInteractions(_ session: RsnapHostSession) throws { try session.send(event: .toolbarItemInvoked(.scroll)) - guard try session.takeNextRequest() == nil else { - fatalError("scroll toolbar invocation should stay disabled") + guard try session.takeNextRequest() == .startScrollCapture else { + fatalError("scroll toolbar invocation should request native scroll capture") } try session.send( diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift index 82e187eb..d3fac39e 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift @@ -101,6 +101,7 @@ final class CaptureHostView: NSView { guard let context = NSGraphicsContext.current?.cgContext else { return } + context.clear(bounds) drawToolbarContent(in: context) } @@ -260,6 +261,7 @@ final class CaptureHostView: NSView { }() private static let pendingHudHexWheel = Array("0123456789ABCDEF") private static let liveChromeLiquidGlassZ: CGFloat = 200 + private static let frozenToolbarLiquidGlassBackdropZ: CGFloat = 295 private static let frozenToolbarLiquidGlassZ: CGFloat = 300 private static let frozenToolbarContentZ: CGFloat = 320 @@ -282,10 +284,12 @@ final class CaptureHostView: NSView { private var settings = NativeHostSettings.defaults private var hudLiquidGlassView: NSView? private var loupeLiquidGlassView: NSView? + private var toolbarLiquidGlassBackdropView: NSImageView? private var toolbarLiquidGlassView: NSView? private var toolbarLiquidGlassContentView: FrozenToolbarRenderView? private var frozenToolbarLiquidGlassVisible = false private var frozenToolbarLiquidGlassContentDrawn = false + private var lastScrollCaptureToolbarBackdropRefreshUptime: TimeInterval = 0 private var trackingAreaRef: NSTrackingArea? private var pointerOverFrozenToolbar = false private var hoveredToolbarAction: ToolbarItemKind? @@ -342,7 +346,7 @@ final class CaptureHostView: NSView { else { return super.hitTest(point) } - return self + return nil } override init(frame frameRect: NSRect) { @@ -594,6 +598,10 @@ final class CaptureHostView: NSView { } func refreshSampleUpdatedLiveChromeNow() { + if scene.mode == .frozen, chrome.scrollMinimapPreview != nil { + refreshScrollCaptureToolbarBackdropNow() + return + } guard scene.mode == .live else { return } @@ -761,6 +769,9 @@ final class CaptureHostView: NSView { performToolbarAction(action) return } + guard chrome.scrollMinimapPreview == nil else { + return + } controller?.beginFrozenInteraction(at: point) syncVisibleCursor() } @@ -814,8 +825,14 @@ final class CaptureHostView: NSView { switch event.charactersIgnoringModifiers?.lowercased() { case "z": if event.modifierFlags.contains(.shift) { + guard toolbarItem(.redo)?.enabled == true else { + return + } controller?.performFrozenRedo() } else { + guard toolbarItem(.undo)?.enabled == true else { + return + } controller?.performFrozenUndo() } return @@ -842,6 +859,9 @@ final class CaptureHostView: NSView { if scene.mode == .frozen, plainFrozenShortcutAvailable(event) { switch event.charactersIgnoringModifiers?.lowercased() { case "c": + guard toolbarItem(.autoCenter)?.enabled == true else { + return + } controller?.performFrozenAutoCenter() return case "r": @@ -850,6 +870,9 @@ final class CaptureHostView: NSView { } controller?.recognizeText() return + case "s": + controller?.startScrollCapture(source: "keyboard_s") + return default: break } @@ -1973,7 +1996,7 @@ final class CaptureHostView: NSView { } var styleKind: FrozenAnnotationStyleToolbarKind? - for item in items where item.selected { + for item in items where item.enabled && item.selected { if let kind = FrozenAnnotationStyleToolbarKind(selectedTool: item.kind) { styleKind = kind break @@ -2139,12 +2162,16 @@ final class CaptureHostView: NSView { private func frozenToolbarScrimExclusionPath(for selection: CGRect) -> CGPath? { guard settings.usesLiquidHudGlass, - frozenToolbarLiquidGlassVisible, - frozenToolbarLiquidGlassContentDrawn, let toolbarFrame = toolbarLayout(for: selection)?.frame else { return nil } + guard + chrome.scrollMinimapPreview != nil + || (frozenToolbarLiquidGlassVisible && frozenToolbarLiquidGlassContentDrawn) + else { + return nil + } let visibleSelection = selection.intersection(bounds) if visibleSelection.isNull == false, toolbarFrame.intersects(visibleSelection) { return nil @@ -2172,26 +2199,27 @@ final class CaptureHostView: NSView { private func visibleToolbarItems() -> [ToolbarItem] { var items: [ToolbarItem] = [] + let scrollCaptureActive = chrome.scrollMinimapPreview != nil for originalItem in scene.toolbarItems { var item = originalItem switch item.kind { - case .pen, .arrow, .mosaic, .spotlight, .text: - item.enabled = true + case .pointer, .pen, .arrow, .mosaic, .spotlight, .text: + item.enabled = originalItem.enabled && !scrollCaptureActive case .undo: - item.enabled = chrome.frozenOverlay.canUndo + item.enabled = chrome.frozenOverlay.canUndo && !scrollCaptureActive case .redo: - item.enabled = chrome.frozenOverlay.canRedo + item.enabled = chrome.frozenOverlay.canRedo && !scrollCaptureActive case .autoCenter: item.enabled = scene.frozenSelection != nil && !chrome.frozenOverlay.keepsFrozenSelectionFixed + && !scrollCaptureActive case .scroll: - guard controller?.scrollCaptureToolbarEnabled == true else { - continue - } item.enabled = controller?.scrollCaptureToolbarEnabled ?? false - default: - break + case .ocr: + item.enabled = originalItem.enabled && !scrollCaptureActive + case .copy, .save: + item.enabled = originalItem.enabled } items.append(item) } @@ -2199,7 +2227,7 @@ final class CaptureHostView: NSView { } private func toolbarItem(_ kind: ToolbarItemKind) -> ToolbarItem? { - scene.toolbarItems.first(where: { $0.kind == kind }) + visibleToolbarItems().first(where: { $0.kind == kind }) } private func toolbarAction(at point: CGPoint) -> ToolbarItemKind? { @@ -3257,6 +3285,10 @@ final class CaptureHostView: NSView { case .live: return controller?.backgroundPatch(in: globalFrame) case .frozen: + if chrome.scrollMinimapPreview != nil { + return controller?.backgroundPatch(in: globalFrame) + ?? frozenDisplayPatch(in: globalFrame) + } return frozenDisplayPatch(in: globalFrame) case .hidden: return nil @@ -3344,6 +3376,17 @@ final class CaptureHostView: NSView { view.layer?.zPosition = zPosition } + private func configureFrozenToolbarBackdropView(_ view: NSImageView) { + view.isHidden = true + view.imageScaling = .scaleAxesIndependently + view.wantsLayer = true + view.layer?.backgroundColor = NSColor.clear.cgColor + view.layer?.cornerRadius = CaptureChrome.hudCornerRadius + view.layer?.masksToBounds = true + view.layer?.isOpaque = false + view.layer?.zPosition = Self.frozenToolbarLiquidGlassBackdropZ + } + private func configureFrozenToolbarContentView(_ view: FrozenToolbarRenderView) { view.isHidden = true view.wantsLayer = true @@ -3437,6 +3480,8 @@ final class CaptureHostView: NSView { if activeView.frame != frame { activeView.frame = frame } + activeView.needsLayout = true + activeView.layoutSubtreeIfNeeded() activeView.isHidden = false } @@ -3461,10 +3506,25 @@ final class CaptureHostView: NSView { ensureFrozenToolbarContentView(above: createdView) } + @discardableResult + private func ensureFrozenToolbarBackdropView(below glassView: NSView) -> NSImageView { + if let toolbarLiquidGlassBackdropView { + addSubview(toolbarLiquidGlassBackdropView, positioned: .below, relativeTo: glassView) + toolbarLiquidGlassBackdropView.layer?.zPosition = Self.frozenToolbarLiquidGlassBackdropZ + return toolbarLiquidGlassBackdropView + } + let backdropView = NSImageView(frame: .zero) + configureFrozenToolbarBackdropView(backdropView) + addSubview(backdropView, positioned: .below, relativeTo: glassView) + toolbarLiquidGlassBackdropView = backdropView + return backdropView + } + @discardableResult private func ensureFrozenToolbarContentView(above glassView: NSView) -> FrozenToolbarRenderView { if let toolbarLiquidGlassContentView { + addSubview(toolbarLiquidGlassContentView, positioned: .above, relativeTo: glassView) toolbarLiquidGlassContentView.layer?.zPosition = Self.frozenToolbarContentZ return toolbarLiquidGlassContentView } @@ -3475,6 +3535,43 @@ final class CaptureHostView: NSView { return contentView } + private func updateFrozenToolbarBackdrop( + for toolbarFrame: CGRect, + below glassView: NSView, + preparingFirstVisibleToolbar: Bool + ) { + guard chrome.scrollMinimapPreview != nil, + let globalFrame = globalRect(from: toolbarFrame), + let patch = + controller?.streamPatch(in: globalFrame) + ?? controller?.backgroundPatch(in: globalFrame) + else { + toolbarLiquidGlassBackdropView?.isHidden = true + toolbarLiquidGlassBackdropView?.image = nil + return + } + let backdropView = ensureFrozenToolbarBackdropView(below: glassView) + if backdropView.frame != toolbarFrame { + backdropView.frame = toolbarFrame + } + backdropView.image = NSImage(cgImage: patch, size: toolbarFrame.size) + backdropView.isHidden = preparingFirstVisibleToolbar + } + + func refreshScrollCaptureToolbarBackdropNow() { + guard settings.usesLiquidHudGlass else { + return + } + let now = ProcessInfo.processInfo.systemUptime + let interval = NativeHostDisplayRefresh.frameInterval( + forTargetFramesPerSecond: currentDisplayTargetFramesPerSecond()) + guard now - lastScrollCaptureToolbarBackdropRefreshUptime >= interval else { + return + } + lastScrollCaptureToolbarBackdropRefreshUptime = now + updateFrozenToolbarLiquidGlassView() + } + private func localAnnotationStyleLayout( _ layout: FrozenAnnotationStyleLayout, relativeTo toolbarFrame: CGRect @@ -3550,13 +3647,23 @@ final class CaptureHostView: NSView { return } toolbarLiquidGlassView.layer?.zPosition = Self.frozenToolbarLiquidGlassZ + let preparingFirstVisibleToolbar = + !wasVisible || !frozenToolbarLiquidGlassVisible || !frozenToolbarLiquidGlassContentDrawn + if preparingFirstVisibleToolbar { + toolbarLiquidGlassView.isHidden = true + } + updateFrozenToolbarBackdrop( + for: layout.frame, + below: toolbarLiquidGlassView, + preparingFirstVisibleToolbar: preparingFirstVisibleToolbar + ) let contentView = ensureFrozenToolbarContentView(above: toolbarLiquidGlassView) let frameChanged = contentView.frame != layout.frame if contentView.frame != layout.frame { contentView.frame = layout.frame contentView.needsDisplay = true } - contentView.isHidden = false + contentView.isHidden = preparingFirstVisibleToolbar let changed = contentView.update( theme: chromeTheme(), settings: settings, @@ -3582,6 +3689,16 @@ final class CaptureHostView: NSView { if frameChanged || changed || !wasVisible || !frozenToolbarLiquidGlassContentDrawn { contentView.display() } + if preparingFirstVisibleToolbar { + CATransaction.begin() + CATransaction.setDisableActions(true) + toolbarLiquidGlassView.isHidden = false + if chrome.scrollMinimapPreview != nil, toolbarLiquidGlassBackdropView?.image != nil { + toolbarLiquidGlassBackdropView?.isHidden = false + } + contentView.isHidden = false + CATransaction.commit() + } frozenToolbarLiquidGlassVisible = true frozenToolbarLiquidGlassContentDrawn = true if wasVisible == false { @@ -3593,6 +3710,8 @@ final class CaptureHostView: NSView { let wasVisible = frozenToolbarLiquidGlassVisible frozenToolbarLiquidGlassVisible = false frozenToolbarLiquidGlassContentDrawn = false + toolbarLiquidGlassBackdropView?.isHidden = true + toolbarLiquidGlassBackdropView?.image = nil toolbarLiquidGlassView?.isHidden = true toolbarLiquidGlassContentView?.isHidden = true if wasVisible { diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureOverlayController.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureOverlayController.swift index df6a69be..d519d40b 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureOverlayController.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureOverlayController.swift @@ -30,6 +30,8 @@ final class CaptureOverlayController { ) private let liveChromeBackdrops = LiveChromeBackdropWindowController() private var pendingCaptureStreamPreparation: (() -> Void)? + private var primaryMousePassthroughToken: UInt64 = 0 + private var allMousePassthroughActive = false init( controller: CaptureSessionController, @@ -233,19 +235,91 @@ final class CaptureOverlayController { guard let window = primaryWindow as? CaptureOverlayWindow else { return perform() } - let previousIgnoresMouseEvents = window.ignoresMouseEvents + primaryMousePassthroughToken &+= 1 + let token = primaryMousePassthroughToken window.ignoresMouseEvents = true let result = perform() - DispatchQueue.main.asyncAfter(deadline: .now() + duration) { [weak window] in - window?.ignoresMouseEvents = previousIgnoresMouseEvents + DispatchQueue.main.asyncAfter(deadline: .now() + duration) { [weak self, weak window] in + guard let self, self.primaryMousePassthroughToken == token else { + return + } + window?.ignoresMouseEvents = false + } + return result + } + + func withAllMousePassthrough(duration: TimeInterval, perform: () -> T) -> T { + let visibleWindows = windows.filter(\.isVisible) + guard visibleWindows.isEmpty == false else { + return perform() + } + primaryMousePassthroughToken &+= 1 + let token = primaryMousePassthroughToken + if allMousePassthroughActive == false { + allMousePassthroughActive = true + for window in visibleWindows where window.ignoresMouseEvents == false { + window.ignoresMouseEvents = true + } + NSApp.updateWindows() + } + let result = perform() + DispatchQueue.main.asyncAfter(deadline: .now() + duration) { [weak self] in + guard let self, self.primaryMousePassthroughToken == token else { + return + } + self.allMousePassthroughActive = false + for window in visibleWindows { + window.ignoresMouseEvents = false + } } return result } func setScrollCaptureMousePassthroughActive(_ active: Bool) { + primaryMousePassthroughToken &+= 1 + allMousePassthroughActive = active for window in windows { window.ignoresMouseEvents = active } + guard active == false, let window = primaryWindow as? CaptureOverlayWindow else { + return + } + window.orderFrontRegardless() + window.makeKey() + window.makeFirstResponder(window.hostView) + } + + func refreshScrollCaptureToolbarBackdropNow() { + for window in windows where window.isVisible { + window.hostView.refreshScrollCaptureToolbarBackdropNow() + } + } + + func withOverlayHiddenForScrollTargetAcquisition(perform: () -> T) -> T { + let visibleWindows = windows.filter(\.isVisible) + let previousIgnoresMouseEvents = visibleWindows.map { $0.ignoresMouseEvents } + let previousFocusedWindowNumber = focusedWindowNumber + + for window in visibleWindows { + window.ignoresMouseEvents = true + window.orderOut(nil) + } + NSApp.updateWindows() + RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.08)) + + let result = perform() + + for (index, window) in visibleWindows.enumerated() { + window.orderFrontRegardless() + if previousFocusedWindowNumber == window.windowNumber { + window.makeKey() + window.makeFirstResponder(window.hostView) + (NSApp.delegate as? NativeHostApplicationController)?.window = window + } + window.ignoresMouseEvents = previousIgnoresMouseEvents[index] + } + + return result } func close() { @@ -307,6 +381,18 @@ final class CaptureOverlayController { liveFrameStream.region(in: rect) } + func nextRegionFrame( + in rect: CGRect, + afterFrameSequence: UInt64, + waitForFresh: Bool + ) -> RGBARegionFrameSnapshot? { + liveFrameStream.nextRegionFrame( + in: rect, + afterFrameSequence: afterFrameSequence, + waitForFresh: waitForFresh + ) + } + func updateLivePreviewDemand( point: CGPoint?, settings: NativeHostSettings, @@ -382,6 +468,12 @@ final class CaptureOverlayController { ) } + func scrollCaptureFallbackSource( + near point: CGPoint + ) -> CaptureSessionController.FrozenCaptureJobSource? { + frozenCaptureJobSource(near: point) + } + fileprivate func liveColorSampleSource(near point: CGPoint) -> LiveColorSampleSource? { guard let referenceWindow = windows.first(where: { $0.frame.contains(point) }) @@ -416,7 +508,7 @@ final class CaptureOverlayController { return Self.captureImageBelowOverlay(in: rect, source: source) } - nonisolated fileprivate static func captureImageBelowOverlay( + nonisolated static func captureImageBelowOverlay( in rect: CGRect, source: CaptureSessionController.FrozenCaptureJobSource ) -> CGImage? { diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+FrozenInteraction.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+FrozenInteraction.swift index 3bd51ae5..451e9eee 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+FrozenInteraction.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+FrozenInteraction.swift @@ -22,13 +22,39 @@ extension CaptureSessionController { sendFrozenAction(.recognizeTextRequested, exitAfter: .recognizeText) } - func startScrollCapture() { - guard Self.scrollCaptureEnabled else { - return - } + func startScrollCapture(source: String = "unknown") { let _ = chromeState.frozenOverlay.commitTextEdit( style: chromeState.annotationStyle.textStyle) - sendFrozenAction(.toolbarItemInvoked(.scroll)) + guard scrollCaptureToolbarEnabled else { + let reason = scrollCaptureEntryBlockedReason() + NativeHostTelemetry.captureEvent( + "capture.scroll_capture_entry", + captureID: currentCaptureTelemetryID, + outcome: "blocked", + detail: scrollCaptureEntryDetail(source: source, reason: reason) + ) + try? setHostStatusMessage(scrollCaptureEntryBlockedMessage(reason: reason)) + refreshOverlay() + return + } + NativeHostTelemetry.captureEvent( + "capture.scroll_capture_entry", + captureID: currentCaptureTelemetryID, + outcome: "requested", + detail: scrollCaptureEntryDetail(source: source, reason: "ready") + ) + do { + try beginNativeScrollCapture() + } catch { + NativeHostTelemetry.captureWarning( + "capture.scroll_capture_entry_failed", + captureID: currentCaptureTelemetryID, + stage: source, + error: String(describing: error) + ) + try? setHostStatusMessage("Scroll Capture could not start.") + refreshOverlay() + } } func invokeToolbarItem(_ item: ToolbarItemKind) { @@ -44,12 +70,57 @@ extension CaptureSessionController { case .ocr: sendFrozenAction(.toolbarItemInvoked(item), exitAfter: .recognizeText) case .scroll: - startScrollCapture() + startScrollCapture(source: "toolbar") default: sendFrozenAction(.toolbarItemInvoked(item)) } } + private func scrollCaptureEntryBlockedReason() -> String { + if Self.scrollCaptureEnabled == false { + return "disabled" + } + if scene.mode != .frozen { + return "requires_frozen" + } + if scrollCaptureState != nil { + return "already_active" + } + if currentFrozenSelection() == nil { + return "no_selection" + } + if chromeState.frozenSelectionEditable == false { + return "not_dragged_region" + } + return "unavailable" + } + + private func scrollCaptureEntryBlockedMessage(reason: String) -> String { + switch reason { + case "disabled": + return "Scroll Capture is disabled." + case "already_active": + return "Scroll Capture is already running." + case "not_dragged_region": + return "Scroll Capture requires a dragged region selection." + case "no_selection", "requires_frozen": + return "Select a dragged region before starting Scroll Capture." + default: + return "Scroll Capture is not available for this selection." + } + } + + private func scrollCaptureEntryDetail(source: String, reason: String) -> String { + [ + "source=\(source)", + "reason=\(reason)", + "scene=\(scene.mode)", + "editable=\(chromeState.frozenSelectionEditable)", + "has_selection=\(currentFrozenSelection() != nil)", + "active=\(scrollCaptureState != nil)", + ].joined(separator: " ") + } + func beginFrozenInteraction(at point: CGPoint) { guard scene.mode == .frozen else { pointerMoved(to: point) diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+Live.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+Live.swift index 56c1f96a..82dd9012 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+Live.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+Live.swift @@ -276,6 +276,10 @@ extension CaptureSessionController { overlayController?.streamPatch(in: rect) } + func cachedRegionImage(in rect: CGRect) -> CGImage? { + overlayController?.cachedRegionImage(in: rect) + } + func updateLivePreviewDemand( point: CGPoint?, settings: NativeHostSettings, diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift index 952c4e2f..d70cdce6 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift @@ -1,13 +1,107 @@ import AppKit -import CoreGraphics +@preconcurrency import CoreGraphics import Foundation import RsnapHostBridge +package func scrollCaptureViewportPoint( + for point: CGPoint, + in viewportRect: CGRect, + desktopFrame: CGRect? = nil +) -> CGPoint? { + for candidate in scrollCaptureViewportPointCandidates(for: point, desktopFrame: desktopFrame) + where viewportRect.inclusivelyContains(candidate) { + return candidate + } + return nil +} + +private func scrollCaptureViewportPointCandidates( + for point: CGPoint, + desktopFrame: CGRect? +) -> [CGPoint] { + let desktop = + desktopFrame + ?? NSScreen.screens.reduce(CGRect.null) { partial, screen in + partial.union(screen.frame) + } + guard desktop.isNull == false else { + return [point] + } + let flippedPoint = CGPoint( + x: point.x, + y: desktop.minY + desktop.maxY - point.y + ) + if abs(flippedPoint.y - point.y) <= 0.5 { + return [point] + } + return [point, flippedPoint] +} + +private struct NativeScrollCaptureSampleFrame: Sendable { + let region: RGBARegionSnapshot + let source: String + let frameSequence: UInt64 + let frameAgeMicroseconds: UInt64 + let prefersPairwiseRegistration: Bool +} + +private struct NativeScrollCaptureFallbackRequest: Sendable { + let rect: CGRect + let source: CaptureSessionController.FrozenCaptureJobSource + let frameSequence: UInt64 +} + +private struct NativeScrollCaptureObservation: Sendable { + let sampledFrame: NativeScrollCaptureSampleFrame + let registrationStrategy: String + let result: ScrollObserveResult? + let errorDescription: String? +} + +private let nativeScrollCaptureMinimumHintRowsForHintedRegistration = 1 + +private struct NativeScrollCapturePreviewUpdate: @unchecked Sendable { + let image: CGImage + let exportWidth: Int + let exportHeight: Int + let result: ScrollObserveResult + let viewportHeightPixels: Int +} + +private struct NativeScrollCaptureObservationBatch: Sendable { + let observations: [NativeScrollCaptureObservation] + let preview: NativeScrollCapturePreviewUpdate? + let previewErrorDescription: String? + let previewExportMilliseconds: Double? +} + +private func writeNativeScrollCaptureDebugDump(_ snapshot: RGBARegionSnapshot, name: String) { + guard + let rawDirectory = ProcessInfo.processInfo.environment["RSNAP_SCROLL_CAPTURE_DUMP_DIR"], + rawDirectory.isEmpty == false, + let pngData = try? RsnapExportEncoder.pngData(from: snapshot) + else { + return + } + let directory = URL(fileURLWithPath: rawDirectory, isDirectory: true) + try? FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true + ) + let safeName = name.replacingOccurrences(of: "/", with: "_") + try? pngData.write(to: directory.appendingPathComponent("\(safeName).png")) +} + extension CaptureSessionController { + private static let scrollCaptureForwardedEventMarker: Int64 = 0x5253_4E41_5053_4352 + private static let scrollCapturePreciseWheelDeltaLimit = 120.0 + private static let scrollCaptureLineWheelDeltaLimit = 12.0 + var scrollCaptureToolbarEnabled: Bool { Self.scrollCaptureEnabled && scene.mode == .frozen && scrollCaptureState == nil + && chromeState.frozenSelectionEditable && currentFrozenSelection() != nil } @@ -15,23 +109,56 @@ extension CaptureSessionController { guard Self.scrollCaptureEnabled else { return false } - guard var state = scrollCaptureState else { + guard let state = scrollCaptureState else { return false } - guard state.viewportRect.contains(point) else { + guard + let viewportPoint = scrollCaptureViewportPoint( + for: point, + in: state.viewportRect + ) + else { return false } + let forwardedByRsnap = Self.scrollCaptureEventWasForwardedByRsnap(event) + if forwardedByRsnap { + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelInterceptTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_intercepted", + captureID: currentCaptureTelemetryID, + outcome: "forwarded_echo", + detail: + "source=overlay,deltaX=\(Int(event.scrollingDeltaX.rounded())),deltaY=\(Int(event.scrollingDeltaY.rounded())),x=\(Int(point.x.rounded())),y=\(Int(point.y.rounded()))" + ) + } + scheduleNativeScrollCaptureSample() + return true + } + let targetPoint = CGPoint( - x: point.x.clamped(to: state.viewportRect.minX...state.viewportRect.maxX), - y: point.y.clamped(to: state.viewportRect.minY...state.viewportRect.maxY) + x: viewportPoint.x.clamped(to: state.viewportRect.minX...state.viewportRect.maxX), + y: viewportPoint.y.clamped(to: state.viewportRect.minY...state.viewportRect.maxY) + ) + guard nativeScrollCaptureAcceptsManualInput(state: state) else { + return true + } + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelInterceptTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_intercepted", + captureID: currentCaptureTelemetryID, + detail: + "source=overlay,deltaX=\(Int(event.scrollingDeltaX.rounded())),deltaY=\(Int(event.scrollingDeltaY.rounded())),x=\(Int(point.x.rounded())),y=\(Int(point.y.rounded())),viewportX=\(Int(viewportPoint.x.rounded())),viewportY=\(Int(viewportPoint.y.rounded()))" + ) + } + let posted = forwardNativeScrollCaptureWheel( + event, + at: Self.scrollCapturePostPoint(for: event, fallbackAppKitPoint: targetPoint) ) - let posted = - overlayController?.withPrimaryMousePassthrough( - duration: Self.scrollCaptureForwardingPassthrough - ) { - Self.postScrollWheelEvent(matching: event, at: targetPoint) - } ?? Self.postScrollWheelEvent(matching: event, at: targetPoint) guard posted else { try? setHostStatusMessage("Could not forward scroll input.") @@ -39,23 +166,23 @@ extension CaptureSessionController { return true } - state.sampleGeneration &+= 1 - let generation = state.sampleGeneration - scrollCaptureState = state - DispatchQueue.main.asyncAfter(deadline: .now() + Self.scrollCaptureSampleDelay) { - [weak self] in - self?.observeNativeScrollCaptureFrame(generation: generation) - } + scheduleNativeScrollCaptureSample() return true } func installNativeScrollCaptureMonitor() { removeNativeScrollCaptureMonitor() + NativeHostTelemetry.captureEvent( + "capture.scroll_input_tap", + captureID: currentCaptureTelemetryID, + outcome: "not_used", + detail: "input=selection_passthrough_global_monitor,accessibility_required=false" + ) scrollCaptureGlobalMonitor = NSEvent.addGlobalMonitorForEvents(matching: .scrollWheel) { - [weak self] _ in + [weak self] event in DispatchQueue.main.async { [weak self] in - self?.scheduleNativeScrollCaptureSampleIfPointerIsInViewport() + self?.recordNativeScrollWheelObserved(event, source: "global_monitor") } } } @@ -68,29 +195,206 @@ extension CaptureSessionController { overlayController?.setScrollCaptureMousePassthroughActive(false) } - func scheduleNativeScrollCaptureSampleIfPointerIsInViewport() { - guard let state = scrollCaptureState else { + func recordNativeScrollWheelObserved(_ event: NSEvent, source: String) { + guard var state = scrollCaptureState else { + return + } + let point = NSEvent.mouseLocation + guard + let viewportPoint = scrollCaptureViewportPoint( + for: point, + in: state.viewportRect + ) + else { + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelObservedTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_observed", + captureID: currentCaptureTelemetryID, + outcome: "outside_viewport", + detail: + "source=\(source),deltaX=\(Int(event.scrollingDeltaX.rounded())),deltaY=\(Int(event.scrollingDeltaY.rounded())),x=\(Int(point.x.rounded())),y=\(Int(point.y.rounded())),viewportX=\(Int(state.viewportRect.minX.rounded())),viewportY=\(Int(state.viewportRect.minY.rounded())),viewportWidth=\(Int(state.viewportRect.width.rounded())),viewportHeight=\(Int(state.viewportRect.height.rounded()))" + ) + } return } - guard state.viewportRect.contains(NSEvent.mouseLocation) else { + guard nativeScrollCaptureAcceptsManualInput(state: state) else { + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelObservedTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_observed", + captureID: currentCaptureTelemetryID, + outcome: "manual_input_ignored", + detail: + "source=\(source),deltaX=\(Int(event.scrollingDeltaX.rounded())),deltaY=\(Int(event.scrollingDeltaY.rounded())),x=\(Int(point.x.rounded())),y=\(Int(point.y.rounded())),viewportX=\(Int(viewportPoint.x.rounded())),viewportY=\(Int(viewportPoint.y.rounded()))" + ) + } return } + state.observedWheelCount &+= 1 + scrollCaptureState = state + let forwardedByRsnap = Self.scrollCaptureEventWasForwardedByRsnap(event) + if forwardedByRsnap == false { + recordNativeScrollCaptureMotionHint( + deltaY: abs(Double(event.scrollingDeltaY)), + multiplier: Self.scrollCapturePassthroughWheelMotionHintMultiplier + ) + } + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelObservedTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_observed", + captureID: currentCaptureTelemetryID, + detail: + "source=\(source),count=\(state.observedWheelCount),deltaX=\(Int(event.scrollingDeltaX.rounded())),deltaY=\(Int(event.scrollingDeltaY.rounded())),x=\(Int(point.x.rounded())),y=\(Int(point.y.rounded())),viewportX=\(Int(viewportPoint.x.rounded())),viewportY=\(Int(viewportPoint.y.rounded())),forwardedByRsnap=\(forwardedByRsnap)" + ) + } + overlayController?.refreshScrollCaptureToolbarBackdropNow() scheduleNativeScrollCaptureSample() } - func scheduleNativeScrollCaptureSample() { + func forwardNativeScrollCaptureWheel(_ event: NSEvent, at targetPoint: CGPoint) -> Bool { + let rawDeltaX = Double(event.scrollingDeltaX) + let rawDeltaY = Double(event.scrollingDeltaY) + let maxMagnitude = max(abs(rawDeltaX), abs(rawDeltaY)) + guard maxMagnitude > 0 else { + return false + } + guard rawDeltaY != 0 else { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_forwarded", + captureID: currentCaptureTelemetryID, + outcome: "horizontal_ignored", + detail: + "rawDeltaX=\(Int(rawDeltaX.rounded())),rawDeltaY=\(Int(rawDeltaY.rounded()))" + ) + return true + } + + guard var state = scrollCaptureState else { + return false + } + state.lastForwardedWheelUptime = ProcessInfo.processInfo.systemUptime + scrollCaptureState = state + + let precise = event.hasPreciseScrollingDeltas + let forwardedDeltaY = Self.forwardedScrollDelta(rawDeltaY, precise: precise) + let postedDeltaY = -forwardedDeltaY + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelForwardedTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_forwarded", + captureID: currentCaptureTelemetryID, + detail: + "rawDeltaX=\(Int(rawDeltaX.rounded())),rawDeltaY=\(Int(rawDeltaY.rounded())),forwardedDeltaY=\(Int(forwardedDeltaY.rounded())),postedDeltaY=\(Int(postedDeltaY.rounded())),precise=\(precise)" + ) + } + let postWheelEvent = { + Self.postScrollWheelEvent( + deltaX: 0, + deltaY: postedDeltaY, + precise: precise, + at: targetPoint + ) + } + let posted = + overlayController?.withAllMousePassthrough( + duration: Self.scrollCaptureForwardingPassthrough + ) { + DispatchQueue.main.async { + _ = postWheelEvent() + } + return true + } + ?? postWheelEvent() + if posted { + recordNativeScrollCaptureMotionHint(deltaY: abs(postedDeltaY)) + scheduleNativeScrollCaptureSample() + } + return posted + } + + func recordNativeScrollCaptureMotionHint(deltaY: Double, multiplier: Double = 1) { + guard deltaY > 0, var state = scrollCaptureState else { + return + } + let viewportHeightRows = + state.viewportRect.height * CGFloat(state.viewportPixelsPerPointY) + let maxHintRows = max(Double(viewportHeightRows) * 0.85, 1) + let hintRows = deltaY * state.viewportPixelsPerPointY * max(multiplier, 0) + state.pendingDownwardMotionHintRows = min( + state.pendingDownwardMotionHintRows + hintRows, + maxHintRows + ) + scrollCaptureState = state + } + + func scheduleNativeScrollCaptureSample( + extendingWindowBy window: TimeInterval = + CaptureSessionController.scrollCaptureInputSampleWindow + ) { guard var state = scrollCaptureState else { return } - state.sampleGeneration &+= 1 - let generation = state.sampleGeneration + + let now = ProcessInfo.processInfo.systemUptime + state.sampleUntilUptime = max(state.sampleUntilUptime, now + max(window, 0)) + guard state.sampleLoopScheduled == false, state.sampleProcessing == false else { + scrollCaptureState = state + scheduleNativeScrollCaptureToolbarBackdropRefresh() + return + } + + state.sampleLoopScheduled = true scrollCaptureState = state - DispatchQueue.main.asyncAfter(deadline: .now() + Self.scrollCaptureSampleDelay) { + scheduleNativeScrollCaptureToolbarBackdropRefresh() + DispatchQueue.main.asyncAfter(deadline: .now() + Self.scrollCaptureSampleInterval) { [weak self] in - self?.observeNativeScrollCaptureFrame(generation: generation) + self?.observeNativeScrollCaptureFrame() + } + } + + func scheduleNativeScrollCaptureToolbarBackdropRefresh() { + guard var state = scrollCaptureState, state.toolbarBackdropLoopScheduled == false else { + return + } + state.toolbarBackdropLoopScheduled = true + scrollCaptureState = state + DispatchQueue.main.asyncAfter( + deadline: .now() + Self.scrollCaptureToolbarBackdropRefreshInterval + ) { [weak self] in + self?.refreshNativeScrollCaptureToolbarBackdrop() } } + func refreshNativeScrollCaptureToolbarBackdrop() { + guard var state = scrollCaptureState else { + return + } + state.toolbarBackdropLoopScheduled = false + scrollCaptureState = state + overlayController?.refreshScrollCaptureToolbarBackdropNow() + guard let latestState = scrollCaptureState, + nativeScrollCaptureShouldKeepSampling(state: latestState) + else { + return + } + scheduleNativeScrollCaptureToolbarBackdropRefresh() + } + + func scheduleNativeScrollCaptureSampleIfNeeded() { + guard let latestState = scrollCaptureState, + nativeScrollCaptureShouldKeepSampling(state: latestState) + else { + return + } + scheduleNativeScrollCaptureSample() + } + func beginNativeScrollCapture() throws { guard Self.scrollCaptureEnabled else { try setHostStatusMessage("Scroll capture is temporarily disabled.") @@ -113,25 +417,63 @@ extension CaptureSessionController { return } + guard + let captureSource = overlayController?.scrollCaptureFallbackSource( + near: CGPoint(x: selection.midX, y: selection.midY) + ) + else { + try setHostStatusMessage("Scroll capture could not locate the overlay window.") + refreshOverlay() + return + } + ensureFrozenBaseImageFromDisplayIfNeeded(for: selection) - let baseImage = chromeState.frozenBaseImage ?? frozenBaseImageFromDisplay(for: selection) - guard let baseImage, let baseSnapshot = NativeHostImageBridge.rgbaSnapshot(from: baseImage) + let frozenBaseImage = chromeState.frozenBaseImage ?? frozenBaseImageFromDisplay(for: selection) + guard let frozenBaseImage, + let frozenBaseSnapshot = NativeHostImageBridge.rgbaSnapshot(from: frozenBaseImage) else { try setHostStatusMessage("Scroll capture could not read the selected region.") refreshOverlay() return } - + let liveBaseImage = CaptureOverlayController.captureImageBelowOverlay( + in: selection, + source: captureSource + ) + let liveBaseSnapshot = liveBaseImage.flatMap { + NativeHostImageBridge.rgbaSnapshot(from: $0) + } + let liveBaseMatchesFrozenSize = + liveBaseSnapshot?.width == frozenBaseSnapshot.width + && liveBaseSnapshot?.height == frozenBaseSnapshot.height + let baseImage: CGImage + let baseSnapshot: RGBARegionSnapshot + let baseSource: String + if let liveBaseImage, let liveBaseSnapshot, liveBaseMatchesFrozenSize { + baseImage = liveBaseImage + baseSnapshot = liveBaseSnapshot + baseSource = "below_overlay_capture_region" + } else { + baseImage = frozenBaseImage + baseSnapshot = frozenBaseSnapshot + baseSource = "frozen_display_region" + } + debugDumpNativeScrollCaptureSnapshot(baseSnapshot, name: "base-\(baseSource)") let stitcher = try RsnapScrollCaptureSession( baseImage: baseSnapshot, previewWidthPixels: baseSnapshot.width ) - scrollCaptureState = NativeScrollCaptureState( + var initialState = NativeScrollCaptureState( stitcher: stitcher, - viewportRect: selection + viewportRect: selection, + captureSource: captureSource, + viewportPixelsPerPointY: Double(baseSnapshot.height) / max(Double(selection.height), 1) ) + initialState.sampleUntilUptime = + ProcessInfo.processInfo.systemUptime + Self.scrollCaptureInitialSampleWindow + scrollCaptureState = initialState installNativeScrollCaptureMonitor() - overlayController?.setScrollCaptureMousePassthroughActive(true) + overlayController?.prepareCaptureStreamsNow(trigger: "scroll_capture_start") chromeState.frozenOverlay.reset() chromeState.frozenSelectionEditable = false chromeState.frozenSelectionInteraction = nil @@ -150,55 +492,511 @@ extension CaptureSessionController { viewportTopYPixels: 0, viewportHeightPixels: CGFloat(baseSnapshot.height) ) - try setHostStatusMessage( - "Scroll capture started. Scroll inside the selection, then copy or save.") + try setHostStatusMessage("Scroll capture started. Scroll inside the selection.") + NativeHostTelemetry.captureEvent( + "capture.scroll_capture_started", + captureID: currentCaptureTelemetryID, + detail: + "width=\(baseSnapshot.width),height=\(baseSnapshot.height),x=\(Int(selection.minX.rounded())),y=\(Int(selection.minY.rounded())),mode=manual_universal,baseSource=\(baseSource),liveBaseMatched=\(liveBaseMatchesFrozenSize)" + ) + NativeHostTelemetry.captureEvent( + "capture.scroll_capture_mode", + captureID: currentCaptureTelemetryID, + outcome: "manual_universal", + detail: + "input=selection_passthrough_global_monitor,permission=screen_recording,accessibility_required=false" + ) refreshOverlay() + overlayController?.focusWindow(at: CGPoint(x: selection.midX, y: selection.midY)) + overlayController?.setScrollCaptureMousePassthroughActive(true) + NativeHostTelemetry.captureEvent( + "capture.scroll_input_ready", + captureID: currentCaptureTelemetryID, + detail: + "input=selection_passthrough_global_monitor,overlay=focused,passthrough=window" + ) + scheduleNativeScrollCaptureSample( + extendingWindowBy: Self.scrollCaptureInitialSampleWindow + ) + scheduleNativeScrollCaptureToolbarBackdropRefresh() } - func observeNativeScrollCaptureFrame(generation: UInt64) { - guard let state = scrollCaptureState, generation <= state.sampleGeneration else { + func observeNativeScrollCaptureFrame() { + guard var state = scrollCaptureState else { return } - guard - let sampleImage = overlayController?.backgroundPatch(in: state.viewportRect), - let sample = NativeHostImageBridge.rgbaSnapshot(from: sampleImage) - else { - try? setHostStatusMessage("Scroll capture could not sample the scrolled region.") - refreshOverlay() + guard state.sampleProcessing == false else { + state.sampleLoopScheduled = false + scrollCaptureState = state + scheduleNativeScrollCaptureSampleIfNeeded() + return + } + state.sampleLoopScheduled = false + state.sampleProcessing = true + state.sampleSequence &+= 1 + let sampleSequence = state.sampleSequence + let observedWheelCount = state.observedWheelCount + let sampleUptime = ProcessInfo.processInfo.systemUptime + let previewRefreshDue = + state.lastPreviewRefreshUptime == 0 + || sampleUptime - state.lastPreviewRefreshUptime + >= Self.scrollCapturePreviewRefreshInterval + let motionRowsHint = + state.pendingDownwardMotionHintRows > 0 + ? Int(state.pendingDownwardMotionHintRows.rounded()) + : nil + let stitcher = state.stitcher + let captureID = currentCaptureTelemetryID + scrollCaptureState = state + overlayController?.refreshScrollCaptureToolbarBackdropNow() + + let sampledFrames = nativeScrollCaptureSampleFrames( + in: state.viewportRect, + afterFrameSequence: state.lastStreamFrameSequence + ) + let fallbackRequest = + sampledFrames.isEmpty + && nativeScrollCaptureFallbackReadyForInput(state: state) + && nativeScrollCaptureFallbackAllowed(at: sampleUptime) + ? NativeScrollCaptureFallbackRequest( + rect: state.viewportRect, + source: state.captureSource, + frameSequence: state.lastStreamFrameSequence &+ 1 + ) : nil + guard sampledFrames.isEmpty == false || fallbackRequest != nil else { + if var latestState = scrollCaptureState, + latestState.sampleSequence == sampleSequence + { + latestState.sampleProcessing = false + scrollCaptureState = latestState + } + recordNativeScrollCaptureMissingSample(state: state, sampleSequence: sampleSequence) + scheduleNativeScrollCaptureSampleIfNeeded() return } + for sampledFrame in sampledFrames { + debugDumpNativeScrollCaptureSnapshot( + sampledFrame.region, + name: "sample-\(sampleSequence)-\(sampledFrame.frameSequence)" + ) + } + + enqueueNativeScrollCaptureObservations( + sampledFrames: sampledFrames, + fallbackRequest: fallbackRequest, + stitcher: stitcher, + motionRowsHint: motionRowsHint, + previewRefreshDue: previewRefreshDue, + captureID: captureID, + sampleSequence: sampleSequence, + observedWheelCount: observedWheelCount + ) + } + + private func enqueueNativeScrollCaptureObservations( + sampledFrames: [NativeScrollCaptureSampleFrame], + fallbackRequest: NativeScrollCaptureFallbackRequest?, + stitcher: RsnapScrollCaptureSession, + motionRowsHint: Int?, + previewRefreshDue: Bool, + captureID: UInt64, + sampleSequence: UInt64, + observedWheelCount: UInt64 + ) { + scrollCaptureStitchQueue.async { + [sampledFrames, fallbackRequest, stitcher, motionRowsHint, previewRefreshDue] in + var sampledFrames = sampledFrames + if sampledFrames.isEmpty, + let fallbackRequest, + let image = CaptureOverlayController.captureImageBelowOverlay( + in: fallbackRequest.rect, + source: fallbackRequest.source + ), + let snapshot = NativeHostImageBridge.rgbaSnapshot(from: image) + { + writeNativeScrollCaptureDebugDump( + snapshot, + name: "fallback-\(fallbackRequest.frameSequence)" + ) + sampledFrames.append( + NativeScrollCaptureSampleFrame( + region: snapshot, + source: "below_overlay_capture_region", + frameSequence: fallbackRequest.frameSequence, + frameAgeMicroseconds: 0, + prefersPairwiseRegistration: true + )) + } + let batch = Self.nativeScrollCaptureObservationBatch( + sampledFrames: sampledFrames, + stitcher: stitcher, + motionRowsHint: motionRowsHint, + previewRefreshDue: previewRefreshDue + ) + DispatchQueue.main.async { [weak self] in + self?.finishNativeScrollCaptureObservations( + batch, + captureID: captureID, + sampleSequence: sampleSequence, + observedWheelCount: observedWheelCount, + motionRowsHint: motionRowsHint + ) + } + } + } + + nonisolated private static func nativeScrollCaptureObservationBatch( + sampledFrames: [NativeScrollCaptureSampleFrame], + stitcher: RsnapScrollCaptureSession, + motionRowsHint: Int?, + previewRefreshDue: Bool + ) -> NativeScrollCaptureObservationBatch { + var observations: [NativeScrollCaptureObservation] = [] + var latestPreviewCandidate: NativeScrollCaptureObservation? + for sampledFrame in sampledFrames { + let observation = nativeScrollCaptureObservation( + sampledFrame, + stitcher: stitcher, + motionRowsHint: motionRowsHint + ) + if observation.result?.outcome != .noChange { + latestPreviewCandidate = observation + } + observations.append(observation) + } + let preview = nativeScrollCapturePreviewUpdate( + stitcher: stitcher, + candidate: latestPreviewCandidate, + previewRefreshDue: previewRefreshDue + ) + return NativeScrollCaptureObservationBatch( + observations: observations, + preview: preview.update, + previewErrorDescription: preview.errorDescription, + previewExportMilliseconds: preview.exportMilliseconds + ) + } + + nonisolated private static func nativeScrollCaptureObservation( + _ sampledFrame: NativeScrollCaptureSampleFrame, + stitcher: RsnapScrollCaptureSession, + motionRowsHint: Int? + ) -> NativeScrollCaptureObservation { + let usesHintedRegistration = nativeScrollCaptureUsesHintedRegistration( + for: sampledFrame, + motionRowsHint: motionRowsHint + ) + let registrationStrategy = usesHintedRegistration ? "hinted_motion" : "pairwise" do { - let result = try state.stitcher.observeDownwardFrame(sample) - try refreshNativeScrollCapturePreview( + let result = + if usesHintedRegistration, let motionRowsHint { + try stitcher.observeDownwardFrame( + sampledFrame.region, + motionRowsHint: motionRowsHint, + allowBurstSearch: true + ) + } else { + try stitcher.observeDownwardFrame(sampledFrame.region) + } + return NativeScrollCaptureObservation( + sampledFrame: sampledFrame, + registrationStrategy: registrationStrategy, result: result, - currentViewportSnapshot: sample + errorDescription: nil + ) + } catch { + return NativeScrollCaptureObservation( + sampledFrame: sampledFrame, + registrationStrategy: registrationStrategy, + result: nil, + errorDescription: String(describing: error) + ) + } + } + + nonisolated private static func nativeScrollCaptureUsesHintedRegistration( + for sampledFrame: NativeScrollCaptureSampleFrame, + motionRowsHint: Int? + ) -> Bool { + guard sampledFrame.prefersPairwiseRegistration == false, + let motionRowsHint, + motionRowsHint >= nativeScrollCaptureMinimumHintRowsForHintedRegistration + else { + return false + } + return true + } + + nonisolated private static func nativeScrollCapturePreviewUpdate( + stitcher: RsnapScrollCaptureSession, + candidate: NativeScrollCaptureObservation?, + previewRefreshDue: Bool + ) -> ( + update: NativeScrollCapturePreviewUpdate?, + errorDescription: String?, + exportMilliseconds: Double? + ) { + guard previewRefreshDue, let candidate, let result = candidate.result else { + return (nil, nil, nil) + } + let previewStartedAt = ProcessInfo.processInfo.systemUptime + do { + if let export = try stitcher.exportImage(), + let exportImage = NativeHostImageBridge.cgImage(from: export) + { + return ( + NativeScrollCapturePreviewUpdate( + image: exportImage, + exportWidth: export.width, + exportHeight: export.height, + result: result, + viewportHeightPixels: candidate.sampledFrame.region.height + ), + nil, + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } + return ( + nil, + "scroll preview export returned no image", + NativeHostTelemetry.milliseconds(since: previewStartedAt) ) } catch { + return ( + nil, + String(describing: error), + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } + } + + private func finishNativeScrollCaptureObservations( + _ batch: NativeScrollCaptureObservationBatch, + captureID: UInt64, + sampleSequence: UInt64, + observedWheelCount: UInt64, + motionRowsHint: Int? + ) { + guard var state = scrollCaptureState, + currentCaptureTelemetryID == captureID, + state.sampleSequence == sampleSequence + else { + return + } + state.sampleProcessing = false + scrollCaptureState = state + defer { + scheduleNativeScrollCaptureSampleIfNeeded() + } + guard batch.observations.isEmpty == false else { + recordNativeScrollCaptureMissingSample(state: state, sampleSequence: sampleSequence) + return + } + + for observation in batch.observations { + let sampledFrame = observation.sampledFrame + if let errorDescription = observation.errorDescription { + NativeHostTelemetry.captureWarning( + "capture.scroll_observe_failed", + captureID: captureID, + stage: "observe_frame", + error: errorDescription + ) + try? setHostStatusMessage("Scroll capture could not stitch that frame.") + refreshOverlay() + continue + } + guard let result = observation.result else { + continue + } + if var latestState = scrollCaptureState { + latestState.lastStreamFrameSequence = sampledFrame.frameSequence + if result.outcome == .committed { + latestState.committedSampleCount &+= 1 + latestState.pendingDownwardMotionHintRows = 0 + } else if result.outcome == .unsupportedDirection { + latestState.pendingDownwardMotionHintRows = 0 + } + scrollCaptureState = latestState + } + NativeHostTelemetry.captureEvent( + "capture.scroll_sample_observed", + captureID: captureID, + outcome: scrollObserveOutcomeName(result.outcome), + detail: + "seq=\(sampleSequence),source=\(sampledFrame.source),registration=\(observation.registrationStrategy),frameSeq=\(sampledFrame.frameSequence),frameAgeMicros=\(sampledFrame.frameAgeMicroseconds),motionRowsHint=\(motionRowsHint ?? 0),growthRows=\(result.growthRows),exportHeight=\(result.exportHeight),viewportTopY=\(result.currentViewportTopY),wheelCount=\(observedWheelCount)" + ) + guard result.outcome != .noChange else { + continue + } + if result.outcome == .unsupportedDirection { + try? setHostStatusMessage("Scroll capture only appends downward motion.") + refreshOverlay() + } + } + + if let preview = batch.preview { + do { + try refreshNativeScrollCapturePreview(preview) + if var latestState = scrollCaptureState { + latestState.lastPreviewRefreshUptime = ProcessInfo.processInfo.systemUptime + scrollCaptureState = latestState + } + let exportMs = + batch.previewExportMilliseconds.map { + String(format: "%.2f", $0) + } ?? "0.00" + NativeHostTelemetry.captureEvent( + "capture.scroll_preview_refreshed", + captureID: captureID, + detail: + "seq=\(sampleSequence),exportWidth=\(preview.exportWidth),exportHeight=\(preview.exportHeight),exportMs=\(exportMs)" + ) + } catch { + NativeHostTelemetry.captureWarning( + "capture.scroll_observe_failed", + captureID: captureID, + stage: "refresh_preview", + error: String(describing: error) + ) + try? setHostStatusMessage("Scroll capture could not stitch that frame.") + refreshOverlay() + } + } else if let previewErrorDescription = batch.previewErrorDescription { NativeHostTelemetry.captureWarning( "capture.scroll_observe_failed", - captureID: currentCaptureTelemetryID, - stage: "observe_frame", - error: String(describing: error) + captureID: captureID, + stage: "refresh_preview", + error: previewErrorDescription ) - try? setHostStatusMessage("Scroll capture could not stitch that frame.") - refreshOverlay() } } - func refreshNativeScrollCapturePreview( - result: ScrollObserveResult, - currentViewportSnapshot: RGBARegionSnapshot - ) throws { - guard let state = scrollCaptureState else { + func debugDumpNativeScrollCaptureSnapshot(_ snapshot: RGBARegionSnapshot, name: String) { + writeNativeScrollCaptureDebugDump(snapshot, name: name) + } + + func nativeScrollCaptureAcceptsManualInput(state _: NativeScrollCaptureState) -> Bool { + true + } + + func nativeScrollCaptureShouldKeepSampling(state: NativeScrollCaptureState) -> Bool { + ProcessInfo.processInfo.systemUptime < state.sampleUntilUptime + } + + func recordNativeScrollCaptureMissingSample( + state: NativeScrollCaptureState, + sampleSequence: UInt64 + ) { + let now = ProcessInfo.processInfo.systemUptime + guard now - state.lastMissingSampleStatusUptime > 0.75 else { return } + NativeHostTelemetry.captureEvent( + "capture.scroll_sample_missing", + captureID: currentCaptureTelemetryID, + outcome: "no_live_stream_region", + detail: "seq=\(sampleSequence)" + ) + if var latestState = scrollCaptureState { + latestState.lastMissingSampleStatusUptime = now + scrollCaptureState = latestState + } + try? setHostStatusMessage("Scroll capture is waiting for a stable live screen frame.") + refreshOverlay() + } + + private func nativeScrollCaptureSampleFrames( + in rect: CGRect, + afterFrameSequence: UInt64 + ) -> [NativeScrollCaptureSampleFrame] { + var frames: [NativeScrollCaptureSampleFrame] = [] + var nextAfterFrameSequence = afterFrameSequence + + for _ in 0.. Bool { + guard var state = scrollCaptureState else { + return false + } guard - let export = try state.stitcher.exportImage(), - let exportImage = NativeHostImageBridge.cgImage(from: export) + state.lastFallbackCaptureUptime == 0 + || uptime - state.lastFallbackCaptureUptime + >= Self.scrollCaptureFallbackCaptureInterval else { - try setHostStatusMessage("Scroll capture could not render the stitched image.") - refreshOverlay() + return false + } + state.lastFallbackCaptureUptime = uptime + scrollCaptureState = state + return true + } + + private func nativeScrollCaptureFallbackReadyForInput( + state: NativeScrollCaptureState + ) -> Bool { + state.observedWheelCount > 0 + || state.pendingDownwardMotionHintRows > 0 + || state.committedSampleCount > 0 + } + + private func nativeScrollCaptureShouldLogWheelTelemetry( + _ keyPath: WritableKeyPath + ) -> Bool { + guard var state = scrollCaptureState else { + return false + } + let uptime = ProcessInfo.processInfo.systemUptime + guard + state[keyPath: keyPath] == 0 + || uptime - state[keyPath: keyPath] + >= Self.scrollCaptureWheelTelemetryInterval + else { + return false + } + state[keyPath: keyPath] = uptime + scrollCaptureState = state + return true + } + + func scrollObserveOutcomeName(_ outcome: ScrollObserveOutcome) -> String { + switch outcome { + case .noChange: + return "no_change" + case .previewUpdated: + return "preview_updated" + case .committed: + return "committed" + case .unsupportedDirection: + return "unsupported_direction" + } + } + + private func refreshNativeScrollCapturePreview( + _ preview: NativeScrollCapturePreviewUpdate + ) throws { + guard let state = scrollCaptureState else { return } @@ -208,29 +1006,37 @@ extension CaptureSessionController { chromeState.frozenDisplayFrame = nil chromeState.frozenDisplayImage = nil chromeState.scrollMinimapPreview = ScrollCaptureMinimapSnapshot( - image: exportImage, - exportSizePixels: CGSize(width: CGFloat(export.width), height: CGFloat(export.height)), - viewportTopYPixels: CGFloat(result.currentViewportTopY), - viewportHeightPixels: CGFloat(currentViewportSnapshot.height) + image: preview.image, + exportSizePixels: CGSize( + width: CGFloat(preview.exportWidth), + height: CGFloat(preview.exportHeight) + ), + viewportTopYPixels: CGFloat(preview.result.currentViewportTopY), + viewportHeightPixels: CGFloat(preview.viewportHeightPixels) ) - if result.outcome == .committed { + if preview.result.outcome == .committed { try setHostStatusMessage( - "Scroll capture appended \(result.growthRows) px. Copy or save exports the stitched image." + "Scroll capture appended \(preview.result.growthRows) px. Copy or save exports the stitched image." ) - } else if result.outcome == .unsupportedDirection { + } else if preview.result.outcome == .unsupportedDirection { try setHostStatusMessage("Scroll capture only appends downward motion.") } refreshOverlay() } - static func postScrollWheelEvent(matching event: NSEvent, at point: CGPoint) -> Bool { - let deltaX = Int32(event.scrollingDeltaX.rounded()) - let deltaY = Int32(event.scrollingDeltaY.rounded()) + static func postScrollWheelEvent( + deltaX rawDeltaX: Double, + deltaY rawDeltaY: Double, + precise: Bool, + at point: CGPoint + ) -> Bool { + let deltaX = Int32(rawDeltaX.rounded()) + let deltaY = Int32(rawDeltaY.rounded()) guard deltaX != 0 || deltaY != 0 else { return false } - let units: CGScrollEventUnit = event.hasPreciseScrollingDeltas ? .pixel : .line + let units: CGScrollEventUnit = precise ? .pixel : .line let wheelCount: UInt32 = deltaX == 0 ? 1 : 2 guard let source = CGEventSource(stateID: .hidSystemState), @@ -246,8 +1052,60 @@ extension CaptureSessionController { return false } + scrollEvent.setIntegerValueField( + .eventSourceUserData, + value: Self.scrollCaptureForwardedEventMarker + ) scrollEvent.location = point scrollEvent.post(tap: .cghidEventTap) return true } + + private static func forwardedScrollDelta(_ delta: Double, precise: Bool) -> Double { + let limit = precise ? scrollCapturePreciseWheelDeltaLimit : scrollCaptureLineWheelDeltaLimit + let clamped = delta.clamped(to: -limit...limit) + let rounded = clamped.rounded() + if abs(rounded) >= 1 { + return rounded + } + if abs(delta) > 0 { + return delta > 0 ? 1 : -1 + } + return 0 + } + + private static func scrollCaptureEventWasForwardedByRsnap(_ event: NSEvent) -> Bool { + event.cgEvent?.getIntegerValueField(.eventSourceUserData) + == Self.scrollCaptureForwardedEventMarker + } + + private static func scrollCapturePostPoint( + for event: NSEvent, + fallbackAppKitPoint: CGPoint + ) -> CGPoint { + if let point = event.cgEvent?.location { + return point + } + return scrollCaptureFlippedDesktopPoint(fallbackAppKitPoint) + } + +} + +private func scrollCaptureFlippedDesktopPoint(_ point: CGPoint) -> CGPoint { + let desktop = NSScreen.screens.reduce(CGRect.null) { partial, screen in + partial.union(screen.frame) + } + guard desktop.isNull == false else { + return point + } + return CGPoint( + x: point.x, + y: desktop.minY + desktop.maxY - point.y + ) +} + +extension CGRect { + fileprivate func inclusivelyContains(_ point: CGPoint) -> Bool { + point.x >= minX && point.x <= maxX && point.y >= minY && point.y <= maxY + } } diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift index 7764617e..6f68b057 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift @@ -30,10 +30,18 @@ final class CaptureSessionController: NSObject { static let autoCenterMaxIterations = 6 static let displayFirstFrameWait: TimeInterval = 0.025 static let coldSelfCaptureRecoveryWait: TimeInterval = 3.5 - static let scrollCaptureEnabled = false - static let scrollCaptureForwardingPassthrough: TimeInterval = 0.055 - static let scrollCaptureSampleDelay: TimeInterval = 0.04 - static let liveFrameStreamReleaseGrace: TimeInterval = 1.5 + static let scrollCaptureEnabled = true + static let scrollCaptureForwardingPassthrough: TimeInterval = 0.08 + static let scrollCaptureSampleInterval: TimeInterval = 1.0 / 30.0 + static let scrollCaptureMaxFramesPerSample = 3 + static let scrollCaptureInitialSampleWindow: TimeInterval = 0.35 + static let scrollCaptureInputSampleWindow: TimeInterval = 0.85 + static let scrollCaptureFallbackCaptureInterval: TimeInterval = 0.08 + static let scrollCapturePreviewRefreshInterval: TimeInterval = 0.18 + static let scrollCaptureToolbarBackdropRefreshInterval: TimeInterval = 1.0 / 120.0 + static let scrollCaptureWheelTelemetryInterval: TimeInterval = 0.25 + static let scrollCapturePassthroughWheelMotionHintMultiplier = 3.5 + static let liveFrameStreamReleaseGrace: TimeInterval = 4.0 let settingsStore: NativeHostSettingsStore let liveFrameStream = LiveFrameStreamBroker() @@ -42,6 +50,10 @@ final class CaptureSessionController: NSObject { label: "ink.hack.rsnap.frozen-commit", qos: .userInitiated ) + let scrollCaptureStitchQueue = DispatchQueue( + label: "ink.hack.rsnap.scroll-capture-stitch", + qos: .userInitiated + ) let captureSuccessSound = CaptureSuccessSound.load() let ocrCompletionSound = OcrCompletionSound.load() var session: RsnapHostSession? diff --git a/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift b/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift index 35c76c6a..6c06521f 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift @@ -549,7 +549,24 @@ struct CaptureChromeState { struct NativeScrollCaptureState { let stitcher: RsnapScrollCaptureSession let viewportRect: CGRect - var sampleGeneration: UInt64 = 0 + let captureSource: CaptureSessionController.FrozenCaptureJobSource + let viewportPixelsPerPointY: Double + var sampleLoopScheduled = false + var sampleProcessing = false + var toolbarBackdropLoopScheduled = false + var sampleSequence: UInt64 = 0 + var observedWheelCount: UInt64 = 0 + var committedSampleCount: UInt64 = 0 + var lastStreamFrameSequence: UInt64 = 0 + var lastMissingSampleStatusUptime: TimeInterval = 0 + var lastForwardedWheelUptime: TimeInterval = 0 + var lastFallbackCaptureUptime: TimeInterval = 0 + var lastPreviewRefreshUptime: TimeInterval = 0 + var lastWheelInterceptTelemetryUptime: TimeInterval = 0 + var lastWheelObservedTelemetryUptime: TimeInterval = 0 + var lastWheelForwardedTelemetryUptime: TimeInterval = 0 + var sampleUntilUptime: TimeInterval = 0 + var pendingDownwardMotionHintRows: Double = 0 } struct ScrollCaptureMinimapSnapshot { diff --git a/native/macos-host/Sources/RsnapNativeHostKit/LiveChromeWindows.swift b/native/macos-host/Sources/RsnapNativeHostKit/LiveChromeWindows.swift index 298f68ec..8f900d8f 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/LiveChromeWindows.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/LiveChromeWindows.swift @@ -243,6 +243,13 @@ final class LiveChromeLiquidGlassView: NSView { nil } + override func layout() { + super.layout() + if glassHostView.frame != bounds { + glassHostView.frame = bounds + } + } + override init(frame frameRect: NSRect) { self.glassHostView = NSHostingView( rootView: Self.makeGlassRoot(settings: .defaults)) @@ -267,10 +274,14 @@ final class LiveChromeLiquidGlassView: NSView { func update(settings: NativeHostSettings) { guard currentSettings != settings else { + needsLayout = true + layoutSubtreeIfNeeded() return } currentSettings = settings glassHostView.rootView = Self.makeGlassRoot(settings: settings) + needsLayout = true + layoutSubtreeIfNeeded() } private static func makeGlassRoot(settings: NativeHostSettings) -> AnyView { diff --git a/native/macos-host/Sources/RsnapNativeHostKit/LiveFrameStream.swift b/native/macos-host/Sources/RsnapNativeHostKit/LiveFrameStream.swift index 8fa1ced3..a4a6f1ce 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/LiveFrameStream.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/LiveFrameStream.swift @@ -248,6 +248,31 @@ final class LiveFrameStreamBroker { ) } + func nextRegionFrame( + in rect: CGRect, + afterFrameSequence: UInt64, + waitForFresh: Bool + ) -> RGBARegionFrameSnapshot? { + guard let monitor = monitor(containing: CGPoint(x: rect.midX, y: rect.midY)) else { + return nil + } + stateLock.lock() + let sampler = self.sampler + let mainDisplayHeight = self.mainDisplayHeight + let encodedMonitor = samplerMonitorSnapshot(for: monitor) + stateLock.unlock() + guard let sampler else { + return nil + } + let quartzRect = Self.appKitRectToQuartz(rect, mainDisplayHeight: mainDisplayHeight) + return try? sampler.nextRegionFrame( + monitor: encodedMonitor, + rect: quartzRect, + afterFrameSequence: afterFrameSequence, + waitForFresh: waitForFresh + ) + } + func prime(at point: CGPoint?) { guard let point, let monitor = monitor(containing: point) else { return diff --git a/native/macos-host/Sources/RsnapNativeHostKit/NativeHostSettingsView.swift b/native/macos-host/Sources/RsnapNativeHostKit/NativeHostSettingsView.swift index a563732f..d667de0c 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/NativeHostSettingsView.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/NativeHostSettingsView.swift @@ -526,10 +526,10 @@ private struct OutputInspector: View { private struct PermissionsInspector: View { var body: some View { - let granted = NativePermissions.screenRecordingGranted ? 1 : 0 + let requiredGranted = NativePermissions.screenRecordingGranted ? 1 : 0 VStack(alignment: .leading, spacing: 12) { - PermissionProgressBadge(granted: granted, total: 1) + PermissionProgressBadge(granted: requiredGranted, total: 1) InspectorMetric( title: "Required", value: "1", @@ -537,9 +537,14 @@ private struct PermissionsInspector: View { ) InspectorMetric( title: "Granted", - value: "\(granted)", + value: "\(requiredGranted)", symbolName: "checkmark.seal" ) + InspectorMetric( + title: "Scroll Capture", + value: NativePermissions.screenRecordingGranted ? "Ready" : "Waiting", + symbolName: "arrow.down.to.line.compact" + ) } } } @@ -2120,7 +2125,17 @@ private struct PermissionsSettingsPanel: View { refresh: { refreshID += 1 model.refresh() - } + }, + isGrantedProvider: { + NativePermissions.screenRecordingGranted + }, + titleWhenGranted: "Screen Recording ready", + titleWhenMissing: "Screen Recording access needed", + subtitleWhenGranted: "The native capture host can see the screen.", + subtitleWhenMissing: + "Open System Settings, then drag Rsnap.app into the Screen Recording app list.", + missingBadgeTitle: "Required", + openSettingsHelp: "Open Screen Recording settings" ) } @@ -2149,6 +2164,13 @@ private struct PermissionGrantCard: View { let appIcon: NSImage let openSettings: () -> Void let refresh: () -> Void + let isGrantedProvider: () -> Bool + let titleWhenGranted: String + let titleWhenMissing: String + let subtitleWhenGranted: String + let subtitleWhenMissing: String + let missingBadgeTitle: String + let openSettingsHelp: String @Environment(\.colorScheme) private var colorScheme @State private var didRefresh = false @@ -2173,7 +2195,7 @@ private struct PermissionGrantCard: View { .fixedSize(horizontal: false, vertical: true) .layoutPriority(1) PermissionStateBadge( - title: isGranted ? "Granted" : "Required", + title: isGranted ? "Granted" : missingBadgeTitle, style: isGranted ? .granted : .required ) } @@ -2204,7 +2226,7 @@ private struct PermissionGrantCard: View { } .rsnapGlassButton(prominent: false) .controlSize(.small) - .help("Open Screen Recording settings") + .help(openSettingsHelp) Button(action: refreshStatus) { Label( @@ -2224,18 +2246,18 @@ private struct PermissionGrantCard: View { private var isGranted: Bool { _ = refreshID - return NativePermissions.screenRecordingGranted + return isGrantedProvider() } private var title: String { - isGranted ? "Screen Recording ready" : "Screen Recording access needed" + isGranted ? titleWhenGranted : titleWhenMissing } private var subtitle: String { if isGranted { - return "The native capture host can see the screen." + return subtitleWhenGranted } - return "Open System Settings, then drag Rsnap.app into the Screen Recording app list." + return subtitleWhenMissing } private var iconBackgroundColor: Color { diff --git a/native/macos-host/Sources/RsnapNativeHostKitProbe/main.swift b/native/macos-host/Sources/RsnapNativeHostKitProbe/main.swift index 7a883fa2..2eaa8535 100644 --- a/native/macos-host/Sources/RsnapNativeHostKitProbe/main.swift +++ b/native/macos-host/Sources/RsnapNativeHostKitProbe/main.swift @@ -9,6 +9,7 @@ enum RsnapNativeHostKitProbe { assertScrimExclusionPreservesExistingPixels() assertRoundedExclusionMaskKeepsCornersFilled() assertCaptureFrameEffectExpandsExportCanvas() + assertScrollCaptureViewportPointAcceptsFlippedGlobalMouseCoordinates() let minimapExportSize = CGSize(width: 100, height: 200) guard let rightMinimap = scrollCaptureMinimapPlan( @@ -103,6 +104,33 @@ enum RsnapNativeHostKitProbe { } } + private static func assertScrollCaptureViewportPointAcceptsFlippedGlobalMouseCoordinates() { + let viewport = CGRect(x: 327, y: 941, width: 808, height: 295) + let desktop = CGRect(x: 0, y: 0, width: 3_008, height: 1_692) + let rawPoint = CGPoint(x: 1_006, y: 676) + guard + let viewportPoint = scrollCaptureViewportPoint( + for: rawPoint, + in: viewport, + desktopFrame: desktop + ), + viewportPoint == CGPoint(x: 1_006, y: 1_016) + else { + fatalError("scroll capture should accept top-origin global wheel coordinates") + } + + let nativePoint = CGPoint(x: 1_006, y: 1_016) + guard + scrollCaptureViewportPoint( + for: nativePoint, + in: viewport, + desktopFrame: desktop + ) == nativePoint + else { + fatalError("scroll capture should preserve native bottom-origin wheel coordinates") + } + } + private static func assertCaptureFrameEffectExpandsExportCanvas() { let imageSize = CGSize(width: 320, height: 180) let canvasSize = CaptureFrameEffectRenderer.canvasSize(for: imageSize) diff --git a/packages/rsnap-capture-core/src/session.rs b/packages/rsnap-capture-core/src/session.rs index 2c8a04ca..7fb18245 100644 --- a/packages/rsnap-capture-core/src/session.rs +++ b/packages/rsnap-capture-core/src/session.rs @@ -215,7 +215,11 @@ impl CaptureSessionCore { self.scene.status_message = None; }, ToolbarItemKind::Undo | ToolbarItemKind::Redo | ToolbarItemKind::AutoCenter => {}, - ToolbarItemKind::Scroll => {}, + ToolbarItemKind::Scroll => { + if self.frozen_selection_editable { + self.pending_requests.push_back(HostRequest::StartScrollCapture); + } + }, ToolbarItemKind::Ocr => { if self.config.allow_text_input { self.pending_requests @@ -250,6 +254,9 @@ impl CaptureSessionCore { self.toolbar_item(ToolbarItemKind::AutoCenter, true), ]; + if self.frozen_selection_editable { + items.push(self.toolbar_item(ToolbarItemKind::Scroll, true)); + } if self.config.allow_text_input { items.push(self.toolbar_item(ToolbarItemKind::Ocr, true)); } @@ -559,13 +566,13 @@ mod tests { assert_eq!(session.scene_model().mode, CaptureMode::Frozen); assert_eq!(session.scene_model().cursor_intent, CursorIntent::Grab); - assert_eq!(session.scene_model().toolbar_items.len(), 12); + assert_eq!(session.scene_model().toolbar_items.len(), 13); assert!( session .scene_model() .toolbar_items .iter() - .all(|item| item.kind != ToolbarItemKind::Scroll) + .any(|item| item.kind == ToolbarItemKind::Scroll && item.enabled) ); } @@ -875,14 +882,22 @@ mod tests { } #[test] - fn scroll_toolbar_invocation_is_disabled_for_drag_region() { + fn scroll_toolbar_invocation_requests_scroll_capture_for_drag_region() { let mut session = CaptureSessionCore::with_config(SessionConfig::default()); enter_frozen_with_drag_selection(&mut session, GlobalRect::new(10, 20, 100, 50)); + assert!( + session + .scene_model() + .toolbar_items + .iter() + .any(|item| item.kind == ToolbarItemKind::Scroll && item.enabled) + ); + session.handle_host_event(HostEvent::ToolbarItemInvoked { item: ToolbarItemKind::Scroll }); - assert_eq!(session.pop_host_request(), None); + assert_eq!(session.pop_host_request(), Some(HostRequest::StartScrollCapture)); } #[test] diff --git a/packages/rsnap-host-ffi/include/rsnap_host_ffi.h b/packages/rsnap-host-ffi/include/rsnap_host_ffi.h index 441bc5ca..fdcb3ec1 100644 --- a/packages/rsnap-host-ffi/include/rsnap_host_ffi.h +++ b/packages/rsnap-host-ffi/include/rsnap_host_ffi.h @@ -8,7 +8,7 @@ extern "C" { #endif -#define RSNAP_HOST_FFI_ABI_VERSION 32u +#define RSNAP_HOST_FFI_ABI_VERSION 34u #define RSNAP_TOOLBAR_ITEM_CAPACITY 16u #define RSNAP_STATUS_MESSAGE_CAPACITY 256u #define RSNAP_LIVE_SAMPLE_PATCH_CAPACITY 4096u @@ -511,6 +511,16 @@ enum RsnapStatus rsnap_scroll_session_observe_downward_frame( size_t rgba_len, struct RsnapScrollObserveResult *out_result ); +enum RsnapStatus rsnap_scroll_session_observe_downward_frame_with_motion_hint( + RsnapScrollSessionHandle *handle, + uint32_t width, + uint32_t height, + const uint8_t *rgba, + size_t rgba_len, + uint32_t motion_rows_hint, + uint8_t allow_burst_search, + struct RsnapScrollObserveResult *out_result +); enum RsnapStatus rsnap_scroll_session_take_export_rgba( RsnapScrollSessionHandle *handle, struct RsnapOwnedRgbaRegion *out_region @@ -704,6 +714,16 @@ enum RsnapStatus rsnap_live_sampler_take_region_rgba( struct RsnapRect rect, struct RsnapOwnedRgbaRegion *out_region ); +enum RsnapStatus rsnap_live_sampler_take_next_region_rgba_after_seq( + RsnapLiveSamplerHandle *handle, + struct RsnapMonitorRect monitor, + struct RsnapRect rect, + uint64_t after_frame_seq, + uint8_t wait_for_fresh, + uint64_t *out_frame_seq, + uint64_t *out_frame_age_micros, + struct RsnapOwnedRgbaRegion *out_region +); enum RsnapStatus rsnap_live_sampler_peek_latest_monitor_rgba( RsnapLiveSamplerHandle *handle, struct RsnapMonitorRect monitor, diff --git a/packages/rsnap-host-ffi/src/lib.rs b/packages/rsnap-host-ffi/src/lib.rs index 1f22541d..f286705e 100644 --- a/packages/rsnap-host-ffi/src/lib.rs +++ b/packages/rsnap-host-ffi/src/lib.rs @@ -44,7 +44,7 @@ use rsnap_overlay::scroll_stitching::{ }; /// ABI version exported by the thin C host bridge. -pub const RSNAP_HOST_FFI_ABI_VERSION: u32 = 32; +pub const RSNAP_HOST_FFI_ABI_VERSION: u32 = 34; const RSNAP_TOOLBAR_ITEM_CAPACITY: usize = 16; const RSNAP_STATUS_MESSAGE_CAPACITY: usize = 256; @@ -1441,6 +1441,55 @@ pub unsafe extern "C" fn rsnap_scroll_session_observe_downward_frame( RsnapStatus::Ok } +/// Observes one discrete viewport screenshot with an optional downward motion hint. +/// +/// # Safety +/// +/// `handle` must be a valid pointer returned by `rsnap_scroll_session_create`. +/// `rgba` must point to `rgba_len` readable bytes containing `width * height * 4` +/// row-major RGBA data, and `out_result` must be writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rsnap_scroll_session_observe_downward_frame_with_motion_hint( + handle: *mut RsnapScrollSessionHandle, + width: u32, + height: u32, + rgba: *const u8, + rgba_len: usize, + motion_rows_hint: u32, + allow_burst_search: u8, + out_result: *mut RsnapScrollObserveResult, +) -> RsnapStatus { + let Some(handle) = (unsafe { scroll_session_handle_mut(handle) }) else { + return RsnapStatus::NullHandle; + }; + + if out_result.is_null() { + return RsnapStatus::NullOutput; + } + + let Some(bytes) = (unsafe { rgba_bytes(rgba, rgba_len) }) else { + return RsnapStatus::InvalidInput; + }; + let hint = (motion_rows_hint > 0).then_some(motion_rows_hint); + let outcome = match handle.session.observe_downward_rgba_with_motion_hint( + width, + height, + bytes, + hint, + allow_burst_search != 0, + ) { + Ok(outcome) => outcome, + Err(_err) => return RsnapStatus::InvalidInput, + }; + let export = handle.session.export_image(); + + unsafe { + ptr::write(out_result, encode_scroll_observe_result(outcome, &export, &handle.session)); + } + + RsnapStatus::Ok +} + /// Copies the current committed scroll-capture export into a Rust-owned RGBA buffer. /// /// # Safety @@ -2619,6 +2668,71 @@ pub unsafe extern "C" fn rsnap_live_sampler_take_region_rgba( RsnapStatus::Ok } +/// Transfers ownership of the oldest queued RGBA region newer than `after_frame_seq` +/// to the caller, preserving live-stream frame order. +/// +/// # Safety +/// +/// `handle` must be a valid pointer returned by `rsnap_live_sampler_create`, +/// `out_frame_seq` and `out_frame_age_micros` must be valid writable pointers, and +/// `out_region` must be a valid writable pointer. The caller must later release the +/// returned region buffer with `rsnap_owned_rgba_region_release`. +#[cfg(target_os = "macos")] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rsnap_live_sampler_take_next_region_rgba_after_seq( + handle: *mut RsnapLiveSamplerHandle, + monitor: RsnapMonitorRect, + rect: RsnapRect, + after_frame_seq: u64, + wait_for_fresh: u8, + out_frame_seq: *mut u64, + out_frame_age_micros: *mut u64, + out_region: *mut RsnapOwnedRgbaRegion, +) -> RsnapStatus { + let Some(handle) = (unsafe { live_sampler_handle_mut(handle) }) else { + return RsnapStatus::NullHandle; + }; + + if out_frame_seq.is_null() || out_frame_age_micros.is_null() || out_region.is_null() { + return RsnapStatus::NullOutput; + } + + let Some(frame) = handle.sampler.next_region_rgba_after_seq( + decode_overlay_monitor(monitor), + decode_overlay_point(RsnapPoint { x: rect.x, y: rect.y }), + rect.width, + rect.height, + after_frame_seq, + wait_for_fresh != 0, + ) else { + unsafe { + ptr::write(out_frame_seq, after_frame_seq); + ptr::write(out_frame_age_micros, 0); + ptr::write(out_region, RsnapOwnedRgbaRegion::default()); + } + + return RsnapStatus::Empty; + }; + let mut rgba = frame.region.rgba; + let out = RsnapOwnedRgbaRegion { + width: frame.region.width, + height: frame.region.height, + len: rgba.len(), + capacity: rgba.capacity(), + rgba: rgba.as_mut_ptr(), + }; + + mem::forget(rgba); + + unsafe { + ptr::write(out_frame_seq, frame.frame_seq); + ptr::write(out_frame_age_micros, frame.frame_age_micros); + ptr::write(out_region, out); + } + + RsnapStatus::Ok +} + /// Peeks the latest cached full-monitor RGBA snapshot from the live sampler without waiting /// for a new capture. /// @@ -4789,6 +4903,81 @@ mod tests { } } + #[cfg(target_os = "macos")] + #[test] + fn ffi_scroll_session_blocks_rewind_until_frontier_is_reacquired() { + let base = scroll_frame(16, 128, 0); + let first = scroll_frame(16, 128, 48); + let rewind = scroll_frame(16, 128, 24); + let below_frontier = scroll_frame(16, 128, 36); + let reacquired = scroll_frame(16, 128, 48); + let beyond_frontier = scroll_frame(16, 128, 60); + let handle = + unsafe { crate::rsnap_scroll_session_create(16, 128, base.as_ptr(), base.len(), 16) }; + + assert!(!handle.is_null()); + + let mut result = RsnapScrollObserveResult::default(); + + assert_eq!( + unsafe { + crate::rsnap_scroll_session_observe_downward_frame( + handle, + 16, + 128, + first.as_ptr(), + first.len(), + &mut result, + ) + }, + RsnapStatus::Ok + ); + assert_eq!(result.kind, RsnapScrollObserveOutcomeKind::Committed as u32); + assert_eq!(result.current_viewport_top_y, 48); + assert_eq!(result.export_height, 176); + + for frame in [&rewind, &below_frontier, &reacquired] { + assert_eq!( + unsafe { + crate::rsnap_scroll_session_observe_downward_frame( + handle, + 16, + 128, + frame.as_ptr(), + frame.len(), + &mut result, + ) + }, + RsnapStatus::Ok + ); + assert_ne!(result.kind, RsnapScrollObserveOutcomeKind::Committed as u32); + assert_eq!(result.current_viewport_top_y, 48); + assert_eq!(result.export_height, 176); + } + + assert_eq!( + unsafe { + crate::rsnap_scroll_session_observe_downward_frame( + handle, + 16, + 128, + beyond_frontier.as_ptr(), + beyond_frontier.len(), + &mut result, + ) + }, + RsnapStatus::Ok + ); + assert_eq!(result.kind, RsnapScrollObserveOutcomeKind::Committed as u32); + assert_eq!(result.growth_rows, 12); + assert_eq!(result.current_viewport_top_y, 60); + assert_eq!(result.export_height, 188); + + unsafe { + crate::rsnap_scroll_session_destroy(handle); + } + } + #[test] fn ffi_click_freeze_request_carries_fixed_selection_payload() { let handle = unsafe { crate::rsnap_session_create(default_config()) }; diff --git a/packages/rsnap-overlay/src/host_live_sampling_macos.rs b/packages/rsnap-overlay/src/host_live_sampling_macos.rs index 97af0fcb..5e679f33 100644 --- a/packages/rsnap-overlay/src/host_live_sampling_macos.rs +++ b/packages/rsnap-overlay/src/host_live_sampling_macos.rs @@ -129,6 +129,45 @@ impl HostMacLiveSampler { }) } + #[must_use] + /// Returns the oldest queued RGBA region after `after_frame_seq`. + /// + /// Callers that need scroll-capture continuity should update `after_frame_seq` + /// with the returned frame sequence and drain until this returns `None`. + pub fn next_region_rgba_after_seq( + &mut self, + monitor: MonitorRect, + origin: GlobalPoint, + width: u32, + height: u32, + after_frame_seq: u64, + wait_for_fresh: bool, + ) -> Option { + let rect = clipped_region_rect(monitor, origin, width, height)?; + let rect_px = monitor.local_rect_to_pixels(rect); + let frames = if wait_for_fresh { + self.stream.ordered_rgba_regions_after_seq(monitor, rect_px, after_frame_seq) + } else { + self.stream.ordered_rgba_regions_after_seq_nonblocking( + monitor, + rect_px, + after_frame_seq, + ) + }?; + let frame = frames.into_iter().next()?; + + Some(HostRgbaRegionFrame { + frame_seq: frame.frame_seq, + frame_age_micros: frame.captured_at.elapsed().as_micros().min(u128::from(u64::MAX)) + as u64, + region: HostRgbaRegion { + width: frame.image.width(), + height: frame.image.height(), + rgba: frame.image.into_raw(), + }, + }) + } + #[must_use] /// Returns the latest cached full-monitor RGBA snapshot when one is already warm. /// @@ -164,6 +203,16 @@ pub struct HostRgbaRegion { pub rgba: Vec, } +/// Owned RGBA pixels plus ScreenCaptureKit frame provenance. +pub struct HostRgbaRegionFrame { + /// Monotonic live stream frame sequence. + pub frame_seq: u64, + /// Age of the sampled ScreenCaptureKit frame in microseconds. + pub frame_age_micros: u64, + /// Region pixels for this frame. + pub region: HostRgbaRegion, +} + fn clipped_region_rect( monitor: MonitorRect, origin: GlobalPoint, diff --git a/packages/rsnap-overlay/src/lib.rs b/packages/rsnap-overlay/src/lib.rs index 9f5d06fb..0b60db13 100644 --- a/packages/rsnap-overlay/src/lib.rs +++ b/packages/rsnap-overlay/src/lib.rs @@ -91,6 +91,30 @@ pub mod scroll_stitching { .map(scroll_stitch_observe_outcome_from) } + /// Observes a discrete native screenshot with an optional downward motion hint. + /// + /// Native macOS auto-scroll knows the commanded AX scroll delta before the next + /// stable frame arrives. Feeding that as a hint lets the stitcher resolve bursty + /// or repeated-content frames without relaxing rewind safety for unhinted samples. + pub fn observe_downward_rgba_with_motion_hint( + &mut self, + width: u32, + height: u32, + rgba: &[u8], + motion_rows_hint: Option, + allow_burst_search: bool, + ) -> Result { + let frame = rgba_image_from_bytes(width, height, rgba)?; + + self.inner + .observe_downward_sample_with_motion_hint_and_burst( + frame, + motion_rows_hint, + allow_burst_search, + ) + .map(scroll_stitch_observe_outcome_from) + } + /// Returns the committed stitched export image. #[must_use] pub fn export_image(&self) -> ScrollStitchImage { diff --git a/packages/rsnap-overlay/src/overlay/tests/worker_tick_runtime.rs b/packages/rsnap-overlay/src/overlay/tests/worker_tick_runtime.rs index 3ae3faa2..260dc5bc 100644 --- a/packages/rsnap-overlay/src/overlay/tests/worker_tick_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/tests/worker_tick_runtime.rs @@ -194,7 +194,7 @@ fn maybe_tick_scroll_capture_does_not_double_count_preview_growth_from_same_late #[cfg(target_os = "macos")] #[test] -fn maybe_tick_scroll_capture_worker_path_recovers_after_blocked_overshot_frame() { +fn maybe_tick_scroll_capture_worker_path_pauses_after_blocked_overshot_frame() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -238,13 +238,17 @@ fn maybe_tick_scroll_capture_worker_path_recovers_after_blocked_overshot_frame() tests::drain_scroll_capture_worker_until_idle(&mut session); - assert_eq!(tests::scroll_capture_export_height(&session), 724); - assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + assert_eq!(tests::scroll_capture_export_height(&session), 640); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 0); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().last_block_reason(), + Some("worker_pairwise_requires_committed_reacquire_after_blocked_gap") + ); } #[cfg(target_os = "macos")] #[test] -fn maybe_tick_scroll_capture_worker_path_retries_immediately_after_blocked_overshot_frame_during_fresh_downward_input() +fn maybe_tick_scroll_capture_worker_path_retries_without_tail_after_blocked_overshot_frame_during_fresh_downward_input() { let monitor = tests::test_monitor(); let rect = RectPoints::new(100, 120, 512, 640); @@ -287,14 +291,17 @@ fn maybe_tick_scroll_capture_worker_path_retries_immediately_after_blocked_overs tests::drain_scroll_capture_worker_until_idle(&mut session); - assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); - assert_eq!(tests::scroll_capture_export_height(&session), 724); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 0); + assert_eq!(tests::scroll_capture_export_height(&session), 640); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().last_block_reason(), + Some("worker_pairwise_requires_committed_reacquire_after_blocked_gap") + ); } #[cfg(target_os = "macos")] #[test] -fn maybe_tick_scroll_capture_worker_path_recovers_across_interleaved_no_frame_and_blocked_browser_steps() - { +fn maybe_tick_scroll_capture_worker_path_requires_reacquire_after_blocked_browser_gap() { let monitor = tests::test_monitor(); let rect = RectPoints::new(100, 120, 512, 640); let mut session = OverlaySession::new(); @@ -310,41 +317,29 @@ fn maybe_tick_scroll_capture_worker_path_recovers_across_interleaved_no_frame_an Some(tests::make_browser_like_worker_capture_window(512, 640, 700)), Some(tests::make_browser_like_worker_capture_window(512, 640, 784)), None, - Some(tests::make_browser_like_worker_capture_window(512, 640, 868)), + Some(tests::make_browser_like_worker_capture_window(512, 640, 84)), + Some(tests::make_browser_like_worker_capture_window(512, 640, 168)), ], ); - for expected_top_y in [84_i32, 168, 252] { - let mut attempts = 0_u8; - - while session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y() - < expected_top_y - { - attempts = attempts.saturating_add(1); - - assert!( - attempts <= 4, - "worker path failed to recover to expected_top_y={expected_top_y}" - ); - - tests::set_scroll_capture_input(&mut session, ScrollDirection::Down); + for expected_top_y in [0_i32, 84, 84, 84, 84, 84, 168] { + tests::set_scroll_capture_input(&mut session, ScrollDirection::Down); - session.scroll_capture.last_external_scroll_input_seq = - session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); - session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.scroll_capture.last_external_scroll_input_seq = + session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); - session.maybe_tick_scroll_capture(); + session.maybe_tick_scroll_capture(); - assert!(session.scroll_capture.inflight_request_id.is_some()); + assert!(session.scroll_capture.inflight_request_id.is_some()); - session.scroll_capture.last_external_scroll_input_seq = - session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); - session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = Some(Instant::now()); - session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_external_scroll_input_seq = + session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; - tests::drain_scroll_capture_worker_until_idle(&mut session); - } + tests::drain_scroll_capture_worker_until_idle(&mut session); assert_eq!( session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 2d774d9d..2f7728cf 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -23,9 +23,9 @@ const FINGERPRINT_GRID_COLUMNS: u32 = 12; const FINGERPRINT_GRID_ROWS: u32 = 16; const DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 48; const DOWNWARD_KEYFRAME_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 4; -const DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS: u32 = 24; +const DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS: u32 = 48; const LOCAL_DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 4; -const LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS: u32 = 12; +const LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS: u32 = 48; const DOWNWARD_REGISTRATION_AMBIGUOUS_GAP_ROWS: u32 = 24; const DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS: u32 = 4; const DOWNWARD_REGISTRATION_MIN_OVERLAP_DIVISOR: u32 = 3; @@ -38,7 +38,7 @@ const REPEATED_PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 4; const TINY_OBSERVED_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 2; const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 1; const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MIN_LAST_HINT_ROWS: u32 = 7; -const BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS: u32 = 40; +const BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS: u32 = 1_024; const DOWNWARD_COMMITTED_KEYFRAME_LOCAL_OVERRUN_MAX_ROWS: u32 = 24; const FALLBACK_DOWNWARD_GROWTH_MIN_ROWS: u32 = 8; const FALLBACK_DOWNWARD_GROWTH_MAX_ROWS: u32 = 16; @@ -202,6 +202,7 @@ pub(crate) struct ScrollSession { growth_history: Vec, last_committed_frame: RgbaImage, worker_pairwise_previous_frame: RgbaImage, + worker_pairwise_requires_committed_reacquire: bool, last_sample_frame: RgbaImage, last_sample_fingerprint: Option>, last_downward_observed_frame: RgbaImage, @@ -269,6 +270,7 @@ impl ScrollSession { growth_history: Vec::new(), last_committed_frame: base_frame.clone(), worker_pairwise_previous_frame: base_frame.clone(), + worker_pairwise_requires_committed_reacquire: false, last_sample_frame: base_frame.clone(), last_sample_fingerprint: Some(fingerprint.clone()), last_downward_observed_frame: base_frame, @@ -373,18 +375,43 @@ impl ScrollSession { let fingerprint = scroll_capture_fingerprint(&frame); let previous_worker_frame = self.worker_pairwise_previous_frame.clone(); + if self.worker_pairwise_requires_committed_reacquire { + if frame == self.last_committed_frame { + self.worker_pairwise_requires_committed_reacquire = false; + + return Ok(self.observe_worker_pairwise_no_change( + frame, + fingerprint, + "worker_pairwise_reacquired_last_committed_frame", + )); + } + + return Ok(self.block_worker_pairwise_until_committed_reacquire(frame, fingerprint)); + } if frame == previous_worker_frame { - return Ok(self.observe_worker_pairwise_no_change( - frame, - fingerprint, - "frame_matches_last_committed_frame", - )); + let reason = if frame == self.last_committed_frame { + "frame_matches_last_committed_frame" + } else { + "frame_matches_worker_pairwise_previous_frame" + }; + + return Ok(self.observe_worker_pairwise_no_change(frame, fingerprint, reason)); } let Some(matched) = self::support::classify_vision_downward_sample_motion_against( &previous_worker_frame, &frame, ) else { + if let Some(upward_motion_rows) = + self::support::trusted_pairwise_upward_shift_rows(&previous_worker_frame, &frame) + { + return Ok(self.observe_worker_pairwise_upward_motion( + frame, + fingerprint, + upward_motion_rows, + )); + } + return Ok(self.observe_worker_pairwise_no_change( frame, fingerprint, @@ -398,16 +425,32 @@ impl ScrollSession { matched.motion_rows, WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, ); - let effective_motion_rows = match Self::resolve_worker_pairwise_motion_rows( + + self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, matched.motion_rows, corroborated_shift_rows, + ) + } + + fn observe_resolved_worker_pairwise_downward_motion( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + vision_motion_rows: u32, + corroborated_shift_rows: Option, + ) -> color_eyre::eyre::Result { + let effective_motion_rows = match Self::resolve_worker_pairwise_motion_rows( + vision_motion_rows, + corroborated_shift_rows, ) { Ok(motion_rows) => motion_rows, Err(block_reason) => { return Ok(self.block_worker_pairwise_growth( frame, fingerprint, - matched.motion_rows, + vision_motion_rows, self.current_viewport_top_y, 0, block_reason, @@ -417,7 +460,7 @@ impl ScrollSession { tracing::debug!( op = "scroll_capture.worker_pairwise_motion_resolved", - vision_motion_rows = matched.motion_rows, + vision_motion_rows, corroborated_motion_rows = corroborated_shift_rows, effective_motion_rows, current_viewport_top_y = self.current_viewport_top_y, @@ -466,15 +509,33 @@ impl ScrollSession { ); self.worker_pairwise_previous_frame = frame.clone(); + self.worker_pairwise_requires_committed_reacquire = false; self.clear_preview_only_downward_recovery_carryover(); + if self.resume_frontier_top_y.is_some() { + let outcome = self.observe_downward_motion_while_resume_frontier_active( + frame.clone(), + effective_motion_rows, + true, + )?; + + if !matches!( + outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, .. } + ) { + self.record_last_sample(&frame, fingerprint); + } + + return Ok(outcome); + } + self.apply_growth( frame.clone(), growth_rows, candidate_viewport_top_y, "worker_pairwise_vision", - Some(matched.motion_rows), + Some(vision_motion_rows), Some(effective_motion_rows), None, ) @@ -486,6 +547,26 @@ impl ScrollSession { fingerprint: Vec, reason: &'static str, ) -> ScrollObserveOutcome { + if reason == "worker_pairwise_vision_no_downward_offset" + && frame != self.last_committed_frame + { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + self.worker_pairwise_requires_committed_reacquire = true; + + self.log_decision( + "scroll_capture.worker_pairwise_no_change", + ScrollDirection::Down, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some(reason), + ); + + return ScrollObserveOutcome::NoChange; + } + self.update_worker_pairwise_reference_frame(frame, fingerprint); self.log_decision( "scroll_capture.worker_pairwise_no_change", @@ -499,6 +580,47 @@ impl ScrollSession { ScrollObserveOutcome::NoChange } + fn observe_worker_pairwise_upward_motion( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + motion_rows: u32, + ) -> ScrollObserveOutcome { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + if self.current_viewport_top_y <= 0 && self.resume_frontier_top_y.is_none() { + if frame != self.last_committed_frame { + self.worker_pairwise_requires_committed_reacquire = true; + } + + self.log_decision( + "scroll_capture.worker_pairwise_upward_at_top", + ScrollDirection::Up, + Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), + Some(self.current_viewport_top_y), + Some(0), + Some("worker_pairwise_upward_motion_without_committed_growth"), + ); + + return ScrollObserveOutcome::PreviewUpdated; + } + + self.worker_pairwise_previous_frame = frame.clone(); + + self.observe_upward_rewind(motion_rows); + self.log_decision( + "scroll_capture.worker_pairwise_rewind_armed", + ScrollDirection::Up, + Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), + None, + None, + Some("worker_pairwise_detected_upward_motion"), + ); + + ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } + } + fn block_worker_pairwise_growth( &mut self, frame: RgbaImage, @@ -508,7 +630,13 @@ impl ScrollSession { growth_rows: u32, reason: &'static str, ) -> ScrollObserveOutcome { - self.update_worker_pairwise_reference_frame(frame, fingerprint); + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + if motion_rows > 0 { + self.worker_pairwise_requires_committed_reacquire = true; + } + self.log_decision( "scroll_capture.worker_pairwise_growth_blocked", ScrollDirection::Down, @@ -521,6 +649,25 @@ impl ScrollSession { ScrollObserveOutcome::NoChange } + fn block_worker_pairwise_until_committed_reacquire( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + ) -> ScrollObserveOutcome { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_growth_blocked", + ScrollDirection::Down, + None, + Some(self.current_viewport_top_y), + Some(0), + Some("worker_pairwise_requires_committed_reacquire_after_blocked_gap"), + ); + + ScrollObserveOutcome::NoChange + } + fn resolve_worker_pairwise_motion_rows( vision_motion_rows: u32, corroborated_shift_rows: Option, @@ -3073,6 +3220,7 @@ impl ScrollSession { self.observed_viewport_top_y = previous.viewport_top_y; self.last_committed_frame = previous.frame.clone(); self.worker_pairwise_previous_frame = previous.frame.clone(); + self.worker_pairwise_requires_committed_reacquire = false; self.last_sample_frame = previous.frame.clone(); self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&previous.frame)); self.last_downward_observed_frame = previous.frame.clone(); @@ -3087,6 +3235,7 @@ impl ScrollSession { } else { self.last_committed_frame = self.anchor_frame.clone(); self.worker_pairwise_previous_frame = self.anchor_frame.clone(); + self.worker_pairwise_requires_committed_reacquire = false; self.last_sample_frame = self.anchor_frame.clone(); self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&self.anchor_frame)); self.last_downward_observed_frame = self.anchor_frame.clone(); diff --git a/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs b/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs index 077be644..47da6bae 100644 --- a/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs +++ b/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs @@ -111,10 +111,11 @@ impl ScrollSession { match (classification, upward_veto) { (DownwardRegistration::Matched(down), Some(up)) - if up.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) - <= down.mean_abs_diff_x100 => + if down.motion_rows <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && down.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + >= up.mean_abs_diff_x100 => { - (DownwardRegistration::NoMatch, Some("upward_veto")) + (DownwardRegistration::NoMatch, Some("direction_ambiguous")) }, (DownwardRegistration::NoMatch, _) => (DownwardRegistration::NoMatch, no_match_reason), (other, _) => (other, None), @@ -175,8 +176,9 @@ impl ScrollSession { match (classification, upward_veto) { (DownwardRegistration::Matched(down), Some(up)) - if up.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) - <= down.mean_abs_diff_x100 => + if down.motion_rows <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && down.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + >= up.mean_abs_diff_x100 => { DownwardRegistration::NoMatch }, @@ -304,8 +306,16 @@ impl ScrollSession { LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS, ) .min(max_motion_rows); + let upper_tolerance = + if last_growth_rows >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS { + last_growth_rows.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + } else { + tolerance + } + .min(max_motion_rows); let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); - let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + let max_motion_rows = + last_growth_rows.saturating_add(upper_tolerance).min(max_motion_rows); return Some(min_motion_rows..=max_motion_rows); } @@ -420,8 +430,16 @@ impl ScrollSession { DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS, ) .min(max_motion_rows); + let upper_tolerance = + if last_growth_rows >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS { + last_growth_rows.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + } else { + tolerance + } + .min(max_motion_rows); let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); - let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + let max_motion_rows = + last_growth_rows.saturating_add(upper_tolerance).min(max_motion_rows); return Some(min_motion_rows..=max_motion_rows); } diff --git a/packages/rsnap-overlay/src/scroll_capture/support.rs b/packages/rsnap-overlay/src/scroll_capture/support.rs index 891e1738..b8f15242 100644 --- a/packages/rsnap-overlay/src/scroll_capture/support.rs +++ b/packages/rsnap-overlay/src/scroll_capture/support.rs @@ -221,6 +221,23 @@ pub(super) fn trusted_pairwise_downward_shift_rows_near_motion( } } +pub(super) fn trusted_pairwise_upward_shift_rows( + previous: &RgbaImage, + current: &RgbaImage, +) -> Option { + let up_match = trusted_pairwise_shift_match(previous, current, ScrollDirection::Up)?; + let down_match = trusted_pairwise_shift_match(previous, current, ScrollDirection::Down); + + if down_match.is_some_and(|down_match| { + down_match.mean_abs_diff_x100 + <= up_match.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + }) { + return None; + } + + Some(up_match.motion_rows) +} + pub(super) fn select_downward_viewport_candidate( candidates: &mut [DownwardViewportCandidate], ) -> DownwardViewportResolution { @@ -229,14 +246,13 @@ pub(super) fn select_downward_viewport_candidate( } if let Some(preferred_local) = prefer_local_downward_viewport_candidate(candidates) { + let ambiguity_margin = downward_viewport_competing_margin(candidates, preferred_local); let competing = candidates.iter().copied().find(|candidate| { candidate != &preferred_local && candidate.viewport_top_y.abs_diff(preferred_local.viewport_top_y) >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS && candidate.mean_abs_diff_x100 - <= preferred_local - .mean_abs_diff_x100 - .saturating_add(DIRECTION_WARNING_MARGIN_X100) + <= preferred_local.mean_abs_diff_x100.saturating_add(ambiguity_margin) }); return match competing { @@ -255,11 +271,12 @@ pub(super) fn select_downward_viewport_candidate( }); let preferred = candidates[0]; + let ambiguity_margin = downward_viewport_competing_margin(candidates, preferred); let competing = candidates.iter().copied().skip(1).find(|candidate| { candidate.viewport_top_y.abs_diff(preferred.viewport_top_y) >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS && candidate.mean_abs_diff_x100 - <= preferred.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + <= preferred.mean_abs_diff_x100.saturating_add(ambiguity_margin) }); match competing { @@ -713,6 +730,21 @@ pub(super) fn evenly_spaced_sample(start: u32, end_exclusive: u32, index: u32, c start.saturating_add(numerator as u32).min(end_exclusive.saturating_sub(1)) } +fn downward_viewport_competing_margin( + candidates: &[DownwardViewportCandidate], + preferred: DownwardViewportCandidate, +) -> u32 { + let exact_corroborated = candidates.iter().any(|candidate| { + candidate != &preferred + && candidate.viewport_top_y == preferred.viewport_top_y + && candidate.motion_rows == preferred.motion_rows + && candidate.mean_abs_diff_x100 + <= preferred.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + }); + + if exact_corroborated { 0 } else { DIRECTION_WARNING_MARGIN_X100 } +} + fn classify_pairwise_downward_shift_near_motion( previous: &RgbaImage, current: &RgbaImage, @@ -743,6 +775,32 @@ fn classify_pairwise_downward_shift_near_motion( classify_downward_registration_candidates(&candidates) } +fn trusted_pairwise_shift_match( + previous: &RgbaImage, + current: &RgbaImage, + direction: ScrollDirection, +) -> Option { + if previous.dimensions() != current.dimensions() { + return None; + } + + let (_width, height) = previous.dimensions(); + + if height < 3 { + return None; + } + + let config = worker_pairwise_overlap_search_config(); + let max_shift = max_directional_motion_rows(previous, current, config); + let candidates = + collect_overlap_direction_matches(previous, current, direction, 1..=max_shift, config); + + match classify_downward_registration_candidates(&candidates) { + DownwardRegistration::Matched(matched) => Some(matched), + DownwardRegistration::Ambiguous { .. } | DownwardRegistration::NoMatch => None, + } +} + fn worker_pairwise_overlap_search_config() -> OverlapSearchConfig { OverlapSearchConfig { min_overlap_rows: 24, diff --git a/packages/rsnap-overlay/src/scroll_capture/tests.rs b/packages/rsnap-overlay/src/scroll_capture/tests.rs index 8050f59a..47953520 100644 --- a/packages/rsnap-overlay/src/scroll_capture/tests.rs +++ b/packages/rsnap-overlay/src/scroll_capture/tests.rs @@ -118,7 +118,10 @@ fn assert_worker_pairwise_repeat_between_steps( Err(err) => panic!("{first_reason}: {err:#}"), }; - assert_eq!(no_change_outcome, ScrollObserveOutcome::NoChange); + assert!(matches!( + no_change_outcome, + ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + )); let followup_growth = assert_worker_pairwise_commit( &mut session, @@ -132,7 +135,7 @@ fn assert_worker_pairwise_repeat_between_steps( } #[cfg(target_os = "macos")] -fn assert_worker_pairwise_blocked_recovery( +fn assert_worker_pairwise_blocked_overshot_does_not_commit_tail( base: image::RgbaImage, blocked: image::RgbaImage, followup: image::RgbaImage, @@ -144,15 +147,24 @@ fn assert_worker_pairwise_blocked_recovery( Err(err) => panic!("{reason}: {err:#}"), }; - assert_eq!(no_change_outcome, ScrollObserveOutcome::NoChange); + assert!(matches!( + no_change_outcome, + ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + )); assert_eq!(session.export_image().height(), base.height()); assert_eq!(session.current_viewport_top_y(), 0); - let growth_rows = - assert_worker_pairwise_commit(&mut session, &blocked, followup.clone(), reason); + let followup_outcome = match session.observe_worker_pairwise_vision_frame(followup.clone()) { + Ok(outcome) => outcome, + Err(err) => panic!("{reason}: {err:#}"), + }; - assert_eq!(session.export_image().height(), base.height() + growth_rows); - assert_eq!(session.current_viewport_top_y(), growth_rows_i32(growth_rows)); + assert!( + !matches!(followup_outcome, ScrollObserveOutcome::Committed { .. }), + "{reason}: blocked overshot followup must not commit tail, got {followup_outcome:?}" + ); + assert_eq!(session.export_image().height(), base.height()); + assert_eq!(session.current_viewport_top_y(), 0); } #[test] @@ -222,6 +234,50 @@ fn session_commits_downward_growth_on_first_matching_sample() { assert_eq!(session.export_image().get_pixel(0, 5), &Rgba([60, 0, 0, 255])); } +#[test] +fn session_fails_closed_on_direction_ambiguous_periodic_shift() { + let document = (0..96) + .map(|row| if row % 2 == 0 { [16, 120, 220, 255] } else { [230, 90, 24, 255] }) + .collect::>(); + let base = make_window(&document, 6, 0, 48); + let moved = make_window(&document, 6, 1, 48); + let mut session = ScrollSession::new(base, 320).unwrap(); + let outcome = session.observe_downward_sample(moved).unwrap(); + let telemetry = session.commit_telemetry(); + + assert!(!matches!(outcome, ScrollObserveOutcome::Committed { .. })); + assert_eq!(session.export_image().height(), 48); + assert_eq!(telemetry.observed_sample_registration_reason, Some("direction_ambiguous")); +} + +#[test] +fn session_tracks_large_slowdown_after_first_growth() { + let mut session = ScrollSession::new(make_sparse_textlike_window(256, 240, 0), 320).unwrap(); + + assert_eq!( + session.observe_downward_sample(make_sparse_textlike_window(256, 240, 114)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 114 } + ); + + let outcome = + session.observe_downward_sample(make_sparse_textlike_window(256, 240, 180)).unwrap(); + + assert_eq!( + outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 66 } + ); + + let outcome = + session.observe_downward_sample(make_sparse_textlike_window(256, 240, 314)).unwrap(); + + assert_eq!( + outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 134 } + ); + assert_eq!(session.export_image().height(), 554); + assert_eq!(session.current_viewport_top_y(), 314); +} + #[cfg(target_os = "macos")] #[test] fn worker_pairwise_vision_commits_substantial_downward_growth_with_corroboration() { @@ -347,12 +403,12 @@ fn worker_pairwise_vision_handles_repeated_frame_between_growth_steps() { #[cfg(target_os = "macos")] #[test] -fn worker_pairwise_vision_recovers_after_blocked_overshot_frame() { - assert_worker_pairwise_blocked_recovery( +fn worker_pairwise_vision_does_not_tail_rebase_after_blocked_overshot_frame() { + assert_worker_pairwise_blocked_overshot_does_not_commit_tail( make_browser_like_window(512, 640, 0), make_browser_like_window(512, 640, 760), make_browser_like_window(512, 640, 844), - "pairwise registration should detect the followup step after the blocked overshot", + "pairwise registration must keep the committed frontier after a blocked overshot", ); } @@ -489,13 +545,62 @@ fn worker_pairwise_vision_handles_repeated_browser_like_frame_between_growth_ste #[cfg(target_os = "macos")] #[test] -fn worker_pairwise_vision_browser_like_followup_uses_adjacent_worker_frame() { - assert_worker_pairwise_blocked_recovery( +fn worker_pairwise_vision_browser_like_followup_does_not_commit_tail_after_blocked_overshot() { + assert_worker_pairwise_blocked_overshot_does_not_commit_tail( make_browser_like_window(512, 640, 0), make_browser_like_window(512, 640, 700), make_browser_like_window(512, 640, 784), - "browser-like pairwise registration should use the immediately previous worker frame", + "browser-like pairwise registration must not append only the tail after a blocked overshot", + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn worker_pairwise_vision_blocks_rewind_recovery_until_frontier_is_reacquired() { + let base = make_browser_like_window(512, 640, 0); + let first = make_browser_like_window(512, 640, 100); + let rewind = make_browser_like_window(512, 640, 60); + let below_frontier = make_browser_like_window(512, 640, 80); + let reacquired = make_browser_like_window(512, 640, 100); + let beyond_frontier = make_browser_like_window(512, 640, 120); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + let first_growth = assert_worker_pairwise_commit( + &mut session, + &base, + first, + "initial pairwise registration should commit downward motion", + ); + let height_after_first = session.export_image().height(); + + assert_eq!(first_growth, 100); + assert!(matches!( + session.observe_worker_pairwise_vision_frame(rewind).unwrap(), + ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } + | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.export_image().height(), height_after_first); + assert_eq!(session.current_viewport_top_y(), 100); + assert!(matches!( + session.observe_worker_pairwise_vision_frame(below_frontier).unwrap(), + ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange + | ScrollObserveOutcome::UnsupportedDirection { .. } + )); + assert_eq!(session.export_image().height(), height_after_first); + assert_eq!(session.current_viewport_top_y(), 100); + assert!(matches!( + session.observe_worker_pairwise_vision_frame(reacquired).unwrap(), + ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.export_image().height(), height_after_first); + assert_eq!(session.current_viewport_top_y(), 100); + assert_eq!( + session.observe_worker_pairwise_vision_frame(beyond_frontier).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } ); + assert_eq!(session.export_image().height(), height_after_first + 20); + assert_eq!(session.current_viewport_top_y(), 120); } #[test] diff --git a/scripts/smoke/lib/live-hud.sh b/scripts/smoke/lib/live-hud.sh index 23700b99..210099be 100644 --- a/scripts/smoke/lib/live-hud.sh +++ b/scripts/smoke/lib/live-hud.sh @@ -18,7 +18,7 @@ live_hud_self_check() { return 1 fi - for cmd in osascript swift swiftc python3; do + for cmd in caffeinate osascript swift swiftc python3; do if ! command -v "$cmd" >/dev/null 2>&1; then echo "missing required tool: $cmd" >&2 return 1 @@ -28,6 +28,68 @@ live_hud_self_check() { echo "[smoke] self-check ok" } +live_hud_start_awake_assertion() { + LIVE_HUD_CAFFEINATE_PID="" + caffeinate -u -t "${RSNAP_DISPLAY_WAKE_SECONDS:-5}" >/dev/null 2>&1 || true + caffeinate -dimsu -w "$$" >/dev/null 2>&1 & + LIVE_HUD_CAFFEINATE_PID="$!" +} + +live_hud_stop_awake_assertion() { + if [[ -n "${LIVE_HUD_CAFFEINATE_PID:-}" ]]; then + kill "$LIVE_HUD_CAFFEINATE_PID" >/dev/null 2>&1 || true + wait "$LIVE_HUD_CAFFEINATE_PID" >/dev/null 2>&1 || true + LIVE_HUD_CAFFEINATE_PID="" + fi +} + +live_hud_assert_interactive_session() { + swift - <<'SWIFT' +import CoreGraphics +import Foundation + +func boolValue(_ value: Any?) -> Bool { + if let bool = value as? Bool { + return bool + } + if let number = value as? NSNumber { + return number.boolValue + } + if let int = value as? Int { + return int != 0 + } + return false +} + +guard let session = CGSessionCopyCurrentDictionary() as? [String: Any] else { + fputs("[smoke] FAIL could not read CG session state\n", stderr) + exit(1) +} + +if boolValue(session["CGSSessionScreenIsLocked"]) { + fputs("[smoke] FAIL macOS session is locked; unlock the display before running native smoke\n", stderr) + exit(1) +} +SWIFT +} + +live_hud_assert_shareable_display() { + swift - <<'SWIFT' +import ScreenCaptureKit + +do { + let content = try await SCShareableContent.current + guard content.displays.isEmpty == false else { + fputs("[smoke] FAIL ScreenCaptureKit returned no displays; check unlocked display and Screen Recording permission\n", stderr) + exit(1) + } +} catch { + fputs("[smoke] FAIL ScreenCaptureKit shareable content unavailable: \(error)\n", stderr) + exit(1) +} +SWIFT +} + live_hud_init_environment() { ROOT_DIR="$1" DISPLAY_BOUNDS="${DISPLAY_BOUNDS:-}" @@ -48,9 +110,15 @@ live_hud_focus_rsnap_overlay() { local focus_settle_s="${RSNAP_FOCUS_SETTLE_S:-0.03}" osascript </dev/null || true +tell application "System Events" + set captureWindowCount to 0 + if exists process "RsnapNativeHost" then + tell process "RsnapNativeHost" + repeat with captureWindow in windows + try + set windowSize to size of captureWindow + if (item 1 of windowSize) > 32 or (item 2 of windowSize) > 32 then + set captureWindowCount to captureWindowCount + 1 + end if + end try + end repeat + end tell + else if exists process "Rsnap" then + tell process "Rsnap" + repeat with captureWindow in windows + try + set windowSize to size of captureWindow + if (item 1 of windowSize) > 32 or (item 2 of windowSize) > 32 then + set captureWindowCount to captureWindowCount + 1 + end if + end try + end repeat + end tell + end if + return captureWindowCount +end tell +APPLESCRIPT +} + +live_hud_cancel_capture_if_present() { + if [[ "${RSNAP_SMOKE_CANCEL_ON_EXIT:-1}" != "1" ]]; then + return 0 + fi + local attempt window_count + for attempt in {1..5}; do + window_count="$(live_hud_capture_window_count | tr -d '[:space:]')" + if [[ -z "$window_count" || "$window_count" == "0" ]]; then + return 0 + fi + osascript <<'APPLESCRIPT' >/dev/null 2>&1 || true +tell application "System Events" + if exists process "RsnapNativeHost" then + tell process "RsnapNativeHost" + set frontmost to true + end tell + delay 0.04 + key code 53 + delay 0.04 + key code 53 + else if exists process "Rsnap" then + tell process "Rsnap" + set frontmost to true + end tell + delay 0.04 + key code 53 + delay 0.04 + key code 53 + end if +end tell +APPLESCRIPT + sleep 0.12 + done + window_count="$(live_hud_capture_window_count | tr -d '[:space:]')" + if [[ -n "$window_count" && "$window_count" != "0" ]]; then + echo "[smoke] WARN capture cleanup still sees $window_count Rsnap window(s)" >&2 + fi +} + live_hud_press_tab() { osascript <<'APPLESCRIPT' tell application "System Events" @@ -104,6 +243,26 @@ end tell APPLESCRIPT } +live_hud_start_new_capture() { + osascript <<'APPLESCRIPT' >/dev/null +tell application "System Events" + if exists process "RsnapNativeHost" then + tell process "RsnapNativeHost" + click menu bar item 1 of menu bar 1 + delay 0.06 + click menu item "New Capture" of menu 1 of menu bar item 1 of menu bar 1 + end tell + else + tell process "Rsnap" + click menu bar item 1 of menu bar 1 + delay 0.06 + click menu item "New Capture" of menu 1 of menu bar item 1 of menu bar 1 + end tell + end if +end tell +APPLESCRIPT +} + live_hud_run_mouse_path() { PATH_POINTS="$PATH_POINTS" \ PATH_MODE="$PATH_MODE" \ @@ -113,6 +272,7 @@ live_hud_run_mouse_path() { PATH_DURATION_MS="$PATH_DURATION_MS" \ PATH_RATE_HZ="$PATH_RATE_HZ" \ PATH_CYCLES="$PATH_CYCLES" \ + PATH_HOLD_BEFORE_RELEASE_MS="${PATH_HOLD_BEFORE_RELEASE_MS:-}" \ swift "$(live_hud_cursor_helper)" } diff --git a/scripts/smoke/lib/native-visual-contract-summary.py b/scripts/smoke/lib/native-visual-contract-summary.py index 996ad667..4c7954e4 100644 --- a/scripts/smoke/lib/native-visual-contract-summary.py +++ b/scripts/smoke/lib/native-visual-contract-summary.py @@ -219,6 +219,52 @@ def count_leaked_horizontal_seam( return seam_pixels / sampled, seam_pixels +def toolbar_tint_dominance( + path: str, display_bounds: str, drag_points: str +) -> tuple[float, int, int] | None: + decoded = png_rgb_rows(path) + if decoded is None: + return None + width, height, channels, rows = decoded + left, top, right, bottom = map(float, display_bounds.replace(" ", "").split(",")) + start_raw, end_raw = drag_points.split(";")[:2] + sx, sy = map(float, start_raw.split(",")) + ex, ey = map(float, end_raw.split(",")) + scale_x = width / max(1.0, right - left) + scale_y = height / max(1.0, bottom - top) + min_x = int((min(sx, ex) - left) * scale_x) + max_x = int((max(sx, ex) - left) * scale_x) + max_y = int((max(sy, ey) - top) * scale_y) + margin_x = max(36, int(40 * scale_x)) + toolbar_top = min(height, max_y + max(4, int(4 * scale_y))) + toolbar_bottom = min(height, max_y + max(80, int(96 * scale_y))) + toolbar_left = max(0, min_x - margin_x) + toolbar_right = min(width, max_x + margin_x) + if toolbar_bottom <= toolbar_top or toolbar_right <= toolbar_left: + return 0.0, 0, 0 + + buckets: dict[tuple[int, int, int], int] = {} + sampled = 0 + step = max(1, int(max(scale_x, scale_y))) + for y in range(toolbar_top, toolbar_bottom, step): + row = rows[y] + for x in range(toolbar_left, toolbar_right, step): + sampled += 1 + base = x * channels + red, green, blue = row[base], row[base + 1], row[base + 2] + rgb_max = max(red, green, blue) + rgb_min = min(red, green, blue) + if rgb_max < 120 or rgb_max - rgb_min < 48: + continue + buckets[(red // 16, green // 16, blue // 16)] = ( + buckets.get((red // 16, green // 16, blue // 16), 0) + 1 + ) + if sampled == 0: + return 0.0, 0, 0 + dominant_pixels = max(buckets.values(), default=0) + return dominant_pixels / sampled, dominant_pixels, sampled + + def pixel_luminance(row: bytes, x: int, channels: int) -> float: base = x * channels red, green, blue = row[base], row[base + 1], row[base + 2] @@ -241,6 +287,7 @@ def line_epoch(line: str) -> float | None: expected_mode = os.environ.get("EXPECTED_HUD_GLASS_MODE", "liquid").strip().lower() screenshot_path = os.environ.get("VISUAL_SCREENSHOT_PATH", "").strip() drag_screenshot_path = os.environ.get("VISUAL_DRAG_SCREENSHOT_PATH", "").strip() +frozen_screenshot_paths = os.environ.get("VISUAL_FROZEN_SCREENSHOT_PATHS", "").strip() visual_display_bounds = os.environ.get("VISUAL_DISPLAY_BOUNDS", "").strip() visual_drag_points = os.environ.get("VISUAL_DRAG_POINTS", "").strip() mask_probe_path = os.environ.get("MASK_PROBE_PATH", "").strip() @@ -537,6 +584,29 @@ def line_epoch(line: str) -> float | None: f"(ratio={seam_ratio:.5f}, pixels={seam_pixels})" ) +if frozen_screenshot_paths and visual_display_bounds and visual_drag_points: + for index, frozen_path in enumerate( + [path for path in frozen_screenshot_paths.split(":") if path], start=1 + ): + dominance = toolbar_tint_dominance( + frozen_path, visual_display_bounds, visual_drag_points + ) + if dominance is None: + failures.append(f"frozen screenshot could not be decoded for toolbar tint check: {frozen_path}") + continue + tint_ratio, tint_pixels, sampled_pixels = dominance + max_tint_ratio = threshold("MAX_FROZEN_TOOLBAR_TINT_DOMINANCE", 0.45) + min_tint_pixels = int(threshold("MIN_FROZEN_TOOLBAR_TINT_PIXELS", 1_500)) + print( + "[smoke] frozen_toolbar_tint " + f"index={index} ratio={tint_ratio:.5f} pixels={tint_pixels} sampled={sampled_pixels}" + ) + if tint_ratio > max_tint_ratio and tint_pixels > min_tint_pixels: + failures.append( + "frozen toolbar appears dominated by one tint color " + f"on screenshot {index} (ratio={tint_ratio:.5f}, pixels={tint_pixels})" + ) + if screenshot_path: try: size = os.path.getsize(screenshot_path) diff --git a/scripts/smoke/lib/pasteboard-image-info.swift b/scripts/smoke/lib/pasteboard-image-info.swift new file mode 100644 index 00000000..a65b7db6 --- /dev/null +++ b/scripts/smoke/lib/pasteboard-image-info.swift @@ -0,0 +1,19 @@ +import AppKit +import Foundation + +let waitMs = Int(ProcessInfo.processInfo.environment["PASTEBOARD_WAIT_MS"] ?? "1200") ?? 1_200 +let deadline = Date().addingTimeInterval(TimeInterval(max(0, waitMs)) / 1_000) + +while Date() <= deadline { + if let image = NSImage(pasteboard: .general) { + var proposedRect = CGRect(origin: .zero, size: image.size) + if let cgImage = image.cgImage(forProposedRect: &proposedRect, context: nil, hints: nil) { + print("width=\(cgImage.width) height=\(cgImage.height)") + exit(0) + } + } + usleep(50_000) +} + +fputs("no image found on pasteboard\n", stderr) +exit(1) diff --git a/scripts/smoke/lib/scroll-background-command.swift b/scripts/smoke/lib/scroll-background-command.swift new file mode 100644 index 00000000..cb2cfcbc --- /dev/null +++ b/scripts/smoke/lib/scroll-background-command.swift @@ -0,0 +1,24 @@ +import Foundation + +func readInt(_ key: String, default value: Int) -> Int { + if let raw = ProcessInfo.processInfo.environment[key], let parsed = Int(raw) { + return parsed + } + return value +} + +let count = readInt("SCROLL_COUNT", default: 28) +let deltaY = readInt("SCROLL_DELTA_Y", default: 120) +let intervalMs = readInt("SCROLL_INTERVAL_MS", default: 28) +let name = Notification.Name("ink.hack.rsnap.ScrollSmoke.ScrollBy") +let center = DistributedNotificationCenter.default() + +for _ in 0.. [CGFloat] { + [ + 28, + max(92, width * 0.36), + max(126, width * 0.64), + ] + } + + private func textColumns(width: CGFloat) -> [CGFloat] { + [ + 82, + max(150, width * 0.36 + 54), + max(220, width * 0.64 + 54), + ].filter { $0 < width - 360 } + } +} + +final class ScrollBackgroundDelegate: NSObject, NSApplicationDelegate { + private var window: NSWindow? + private var scrollView: NSScrollView? + private var boundsObserver: NSObjectProtocol? + private let scrollCommandName = Notification.Name("ink.hack.rsnap.ScrollSmoke.ScrollBy") + + func applicationDidFinishLaunching(_: Notification) { + let frame = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1_280, height: 720) + let window = NSWindow( + contentRect: frame, + styleMask: [.borderless], + backing: .buffered, + defer: false + ) + window.backgroundColor = NSColor.white + window.isOpaque = true + window.level = .floating + window.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle] + + let scrollView = NSScrollView(frame: CGRect(origin: .zero, size: frame.size)) + scrollView.autoresizingMask = [.width, .height] + scrollView.hasVerticalScroller = true + scrollView.hasHorizontalScroller = false + scrollView.drawsBackground = true + scrollView.backgroundColor = .white + scrollView.borderType = .noBorder + scrollView.scrollerStyle = .overlay + scrollView.contentView.postsBoundsChangedNotifications = true + + let documentHeight = max(frame.height * 5, 5_760) + let documentView = ScrollDocumentView( + frame: CGRect(x: 0, y: 0, width: frame.width, height: documentHeight) + ) + scrollView.documentView = documentView + window.contentView = scrollView + + DistributedNotificationCenter.default().addObserver( + self, + selector: #selector(handleScrollCommand(_:)), + name: scrollCommandName, + object: nil + ) + boundsObserver = NotificationCenter.default.addObserver( + forName: NSView.boundsDidChangeNotification, + object: scrollView.contentView, + queue: .main + ) { [weak self] _ in + self?.logCurrentOffset() + } + window.orderFrontRegardless() + window.makeKey() + NSApp.activate(ignoringOtherApps: true) + self.window = window + self.scrollView = scrollView + fputs("ready\n", stdout) + fflush(stdout) + } + + @objc + private func handleScrollCommand(_ notification: Notification) { + guard let scrollView else { + return + } + let rawDelta = notification.userInfo?["deltaY"] as? NSNumber + let deltaY = CGFloat(rawDelta?.doubleValue ?? 0) + guard deltaY != 0 else { + return + } + + let clipView = scrollView.contentView + let documentHeight = scrollView.documentView?.bounds.height ?? clipView.bounds.height + let maxY = max(0, documentHeight - clipView.bounds.height) + let nextY = min(max(clipView.bounds.origin.y + deltaY, 0), maxY) + clipView.scroll(to: CGPoint(x: clipView.bounds.origin.x, y: nextY)) + scrollView.reflectScrolledClipView(clipView) + logCurrentOffset() + } + + private func logCurrentOffset() { + guard let scrollView else { + return + } + let offsetY = scrollView.contentView.bounds.origin.y + fputs("offsetY=\(String(format: "%.2f", offsetY))\n", stdout) + fflush(stdout) + } + + func applicationWillTerminate(_: Notification) { + DistributedNotificationCenter.default().removeObserver(self) + if let boundsObserver { + NotificationCenter.default.removeObserver(boundsObserver) + } + } +} + +let app = NSApplication.shared +let delegate = ScrollBackgroundDelegate() +app.setActivationPolicy(.accessory) +app.delegate = delegate +app.run() diff --git a/scripts/smoke/lib/scroll-wheel-burst.swift b/scripts/smoke/lib/scroll-wheel-burst.swift new file mode 100644 index 00000000..a3a47253 --- /dev/null +++ b/scripts/smoke/lib/scroll-wheel-burst.swift @@ -0,0 +1,59 @@ +import ApplicationServices +import Foundation + +func readInt(_ key: String, default value: Int) -> Int { + if let raw = ProcessInfo.processInfo.environment[key], let parsed = Int(raw) { + return parsed + } + return value +} + +func readPoint(_ key: String) -> CGPoint { + let raw = ProcessInfo.processInfo.environment[key] ?? "" + let parts = raw.split(separator: ",") + guard parts.count == 2, + let x = Double(parts[0]), + let y = Double(parts[1]) + else { + fputs("invalid point env for \(key): \(raw)\n", stderr) + exit(2) + } + return CGPoint(x: x, y: y) +} + +let point = readPoint("SCROLL_POINT") +let count = readInt("SCROLL_COUNT", default: 28) +let deltaY = readInt("SCROLL_DELTA_Y", default: -120) +let intervalMs = readInt("SCROLL_INTERVAL_MS", default: 28) +guard let source = CGEventSource(stateID: .hidSystemState) else { + fputs("failed to create CGEventSource\n", stderr) + exit(1) +} + +_ = CGWarpMouseCursorPosition(point) +CGEvent( + mouseEventSource: source, + mouseType: .mouseMoved, + mouseCursorPosition: point, + mouseButton: .left +)?.post(tap: .cghidEventTap) +usleep(120_000) + +for _ in 0..&2 + exit 2 + ;; +esac + +live_hud_self_check +live_hud_start_awake_assertion +live_hud_assert_interactive_session +live_hud_assert_shareable_display +ROOT_DIR="$(live_hud_repo_root)" +live_hud_init_environment "$ROOT_DIR" + +smoke_log() { + printf '[smoke] +%ss %s\n' "$SECONDS" "$*" +} + +PREF_DOMAIN="${RSNAP_PREF_DOMAIN:-ink.hack.rsnap}" +PREF_SNAPSHOT="$(mktemp "${TMPDIR:-/tmp}/rsnap-scroll-prefs.XXXXXX.plist")" +PREF_SNAPSHOT_EXISTS=0 +SCROLL_BACKGROUND_PID="" +SCROLL_BACKGROUND_READY="$(mktemp "${TMPDIR:-/tmp}/rsnap-scroll-bg.XXXXXX.ready")" +SCROLL_COUNT="${SCROLL_COUNT:-14}" +SCROLL_DRIVER="${SCROLL_DRIVER:-wheel}" +SCROLL_START_METHOD="${SCROLL_START_METHOD:-keyboard}" +SCROLL_DELTA_Y="${SCROLL_DELTA_Y:-36}" +SCROLL_INTERVAL_MS="${SCROLL_INTERVAL_MS:-220}" +MIN_SCROLL_COMMITS="${MIN_SCROLL_COMMITS:-3}" +MIN_EXPORT_GROWTH_PX="${MIN_EXPORT_GROWTH_PX:-180}" +APP_POST_VERIFY_SETTLE_S="${APP_POST_VERIFY_SETTLE_S:-0}" +OVERLAY_SETTLE_S="${OVERLAY_SETTLE_S:-0.10}" +POST_FREEZE_SETTLE_S="${POST_FREEZE_SETTLE_S:-0.16}" +POST_COPY_SETTLE_S="${POST_COPY_SETTLE_S:-0.30}" +PATH_CYCLES="${PATH_CYCLES:-1}" +if [[ -z "${POST_SCROLL_SETTLE_S:-}" ]]; then + POST_SCROLL_SETTLE_S=2.20 +fi + +restore_preferences() { + live_hud_stop_awake_assertion + live_hud_cancel_capture_if_present + if [[ -n "$SCROLL_BACKGROUND_PID" ]]; then + kill "$SCROLL_BACKGROUND_PID" >/dev/null 2>&1 || true + fi + if [[ "$PREF_SNAPSHOT_EXISTS" == "1" ]]; then + defaults import "$PREF_DOMAIN" "$PREF_SNAPSHOT" >/dev/null 2>&1 || true + else + for key in captureHotkey hudGlassEnabled hudGlassMode liquidGlassStyle toolbarPlacement loupeSampleSize scrollCaptureAutoScrollEnabled; do + defaults delete "$PREF_DOMAIN" "$key" >/dev/null 2>&1 || true + done + fi + rm -f "$PREF_SNAPSHOT" "$SCROLL_BACKGROUND_READY" +} + +if defaults export "$PREF_DOMAIN" "$PREF_SNAPSHOT" >/dev/null 2>&1; then + PREF_SNAPSHOT_EXISTS=1 +fi +trap restore_preferences EXIT + +wait_ready_file() { + local path="$1" + local attempt + + for attempt in {1..80}; do + if grep -q '^ready$' "$path" >/dev/null 2>&1; then + return 0 + fi + sleep 0.05 + done + return 1 +} + +assert_scroll_background_moved() { + local last_offset + + last_offset="$( + awk -F= '/^offsetY=/ { value=$2 } END { if (value != "") print value }' "$SCROLL_BACKGROUND_READY" + )" + if [[ -z "$last_offset" ]]; then + echo "[smoke] FAIL scroll background did not report any movement" >&2 + return 1 + fi + smoke_log "scroll background offsetY=$last_offset" + python3 - "$last_offset" <<'PY' +import sys + +offset = float(sys.argv[1]) +if offset <= 0: + raise SystemExit("[smoke] FAIL scroll background offset did not advance") +PY +} + +click_scroll_toolbar_icon() { + local point + + point="$( + python3 - "$DRAG_POINTS" "$DISPLAY_BOUNDS" <<'PY' +import os +import sys + +drag_points, display_bounds = sys.argv[1:3] +start_raw, end_raw = drag_points.split(";") +x1, y1 = map(float, start_raw.split(",")) +x2, y2 = map(float, end_raw.split(",")) +left, top, right, bottom = map(float, display_bounds.replace(" ", "").split(",")) + +selection_min_x = min(x1, x2) +selection_max_x = max(x1, x2) +# The mouse helper posts Quartz-style display coordinates, while AppKit lays out +# the toolbar in bottom-left screen coordinates. +selection_min_y = bottom - max(y1, y2) +selection_max_y = bottom - min(y1, y2) +selection_mid_x = (selection_min_x + selection_max_x) / 2 + +scale = min(1.0, 30.0 / ((5.0 * 2.0) + 24.0)) +button_size = 24.0 * scale +item_spacing = 4.0 * scale +horizontal_padding = 12.0 * scale +vertical_padding = 5.0 * scale +gap = 10.0 * scale +screen_margin = 10.0 + +# Current frozen toolbar order has scroll after auto-center. The 12.5 default +# lands inside the scroll button for the 12- and 13-item variants. +item_count = float(os.environ.get("SCROLL_TOOLBAR_ITEM_COUNT", "12.5")) +scroll_index = float(os.environ.get("SCROLL_TOOLBAR_SCROLL_INDEX", "9")) +primary_content_width = item_count * button_size + max(0.0, item_count - 1.0) * item_spacing +width = primary_content_width + horizontal_padding * 2.0 +height = vertical_padding * 2.0 + button_size + +desired_y = selection_max_y + gap +placed_above = desired_y + height > bottom - screen_margin +if placed_above: + frame_y = max(top + screen_margin, selection_min_y - gap - height) +else: + frame_y = min(bottom - screen_margin - height, desired_y) +frame_min_x = max(left + screen_margin, min(selection_mid_x - width / 2.0, right - screen_margin - width)) +cursor_x = frame_min_x + horizontal_padding +center_x = cursor_x + scroll_index * (button_size + item_spacing) + button_size / 2.0 +center_y = bottom - (frame_y + height / 2.0) +print(f"{round(center_x)},{round(center_y)}") +PY + )" + smoke_log "clicking scroll toolbar icon at $point" + PATH_POINTS="$point;$point" \ + PATH_MODE="click-point" \ + PATH_DRIVER="${PATH_DRIVER:-event}" \ + PATH_SEGMENT_STEPS="${PATH_SEGMENT_STEPS:-1}" \ + PATH_STEP_DELAY_MS="${PATH_STEP_DELAY_MS:-0}" \ + PATH_DURATION_MS="${PATH_DURATION_MS:-0}" \ + PATH_RATE_HZ="${PATH_RATE_HZ:-120}" \ + PATH_CYCLES="${PATH_CYCLES:-1}" \ + live_hud_run_mouse_path +} + +press_capture_hotkey() { + live_hud_start_new_capture +} + +press_plain_s() { + osascript <<'APPLESCRIPT' >/dev/null +tell application "System Events" + key code 1 +end tell +APPLESCRIPT +} + +press_space() { + osascript <<'APPLESCRIPT' >/dev/null +tell application "System Events" + key code 49 +end tell +APPLESCRIPT +} + +configure_preferences() { + defaults write "$PREF_DOMAIN" captureHotkey -string Option-X + defaults write "$PREF_DOMAIN" hudGlassEnabled -bool true + defaults write "$PREF_DOMAIN" hudGlassMode -string liquid_glass + defaults write "$PREF_DOMAIN" liquidGlassStyle -string clear + defaults write "$PREF_DOMAIN" toolbarPlacement -string bottom + defaults write "$PREF_DOMAIN" loupeSampleSize -string small + defaults delete "$PREF_DOMAIN" scrollCaptureAutoScrollEnabled >/dev/null 2>&1 || true +} + +start_scroll_background() { + rm -f "$SCROLL_BACKGROUND_READY" + swift "$SCRIPT_DIR/lib/scroll-background-window.swift" >"$SCROLL_BACKGROUND_READY" & + SCROLL_BACKGROUND_PID="$!" + if ! wait_ready_file "$SCROLL_BACKGROUND_READY"; then + echo "[smoke] FAIL scroll background did not become ready" >&2 + return 1 + fi +} + +parse_telemetry() { + local log_path="$1" + + python3 - "$log_path" "$MIN_SCROLL_COMMITS" "$MIN_EXPORT_GROWTH_PX" "$SCROLL_START_METHOD" <<'PY' +import re +import sys + +path, min_commits_raw, min_growth_raw, start_method = sys.argv[1:5] +min_commits = int(min_commits_raw) +min_growth = int(min_growth_raw) +expected_start_source = { + "keyboard": "keyboard_s", + "toolbar": "toolbar", +}[start_method] +text = open(path, "r", encoding="utf-8", errors="replace").read() + +froze = "event=capture_timing.freeze_commit" in text +handoff = "event=capture_timing.frozen_first_display_handoff" in text +entry_started = bool( + re.search( + rf"event=capture\.scroll_capture_entry\b[^\n]*outcome=requested[^\n]*source={expected_start_source}\b", + text, + ) +) +started = "event=capture.scroll_capture_started" in text +manual_mode = bool( + re.search( + r"event=capture\.scroll_capture_mode\b[^\n]*outcome=manual_universal\b", + text, + ) +) +tap_not_used = bool( + re.search( + r"event=capture\.scroll_input_tap\b[^\n]*outcome=not_used\b", + text, + ) +) +wheel_intercepted = bool( + re.search( + r"event=capture\.scroll_wheel_intercepted\b[^\n]*source=overlay\b", + text, + ) +) +wheel_observed = bool( + re.search( + r"event=capture\.scroll_wheel_observed\b[^\n]*source=global_monitor\b", + text, + ) +) +wheel_input_seen = wheel_intercepted or wheel_observed +sampled = "event=capture.scroll_sample_observed" in text +started_match = re.search(r"event=capture\.scroll_capture_started\b[^\n]*height=([0-9]+)", text) +base_height = int(started_match.group(1)) if started_match else 0 +commits = re.findall( + r"event=capture\.scroll_sample_observed\b[^\n]*outcome=committed\b[^\n]*", + text, +) +heights = [] +for line in re.findall(r"event=capture\.scroll_sample_observed\b[^\n]*", text): + match = re.search(r"exportHeight=([0-9]+)", line) + if match: + heights.append(int(match.group(1))) +fallback = "source=below_overlay_capture" in text +missing = "event=capture.scroll_sample_missing" in text +auto_event = bool(re.search(r"event=capture\.scroll_auto_", text)) +max_height = max(heights) if heights else 0 +growth = max_height - base_height + +print( + f"[smoke] telemetry froze={froze} handoff={handoff} " + f"entry_started={entry_started} start_source={expected_start_source} " + f"started={started} manual_mode={manual_mode} sampled={sampled} commits={len(commits)} " + f"tap_not_used={tap_not_used} wheel_intercepted={wheel_intercepted} " + f"wheel_observed={wheel_observed} " + f"max_export_height={max_height} base_height={base_height} growth={growth} " + f"missing_live_frame={missing} auto_event={auto_event}" +) + +failures = [] +if not froze: + failures.append("drag selection did not freeze") +if not handoff: + failures.append("frozen first display handoff was not recorded") +if not entry_started: + failures.append(f"scroll capture did not start from {expected_start_source}") +if not started: + failures.append("scroll capture did not start") +if not manual_mode: + failures.append("scroll capture did not use universal manual mode") +if not tap_not_used: + failures.append("scroll capture did not use overlay-local wheel forwarding") +if not wheel_input_seen: + failures.append("scroll capture did not receive wheel input") +if not sampled: + failures.append("scroll capture did not sample") +if base_height <= 0: + failures.append("scroll capture start height was not recorded") +if len(commits) < min_commits: + failures.append(f"committed growth count {len(commits)} < {min_commits}") +if growth < min_growth: + failures.append(f"export growth {growth}px < {min_growth}px") +if auto_event: + failures.append("unexpected legacy auto-scroll telemetry") + +if failures: + for failure in failures: + print(f"[smoke] FAIL {failure}", file=sys.stderr) + sys.exit(1) +PY +} + +if [[ -z "${DISPLAY_BOUNDS:-}" ]]; then + DISPLAY_BOUNDS="$(live_hud_read_main_display_bounds | tr -d ' ')" +fi + +if [[ -z "${DRAG_POINTS:-}" ]]; then + DRAG_POINTS="$( + python3 - "$DISPLAY_BOUNDS" <<'PY' +import sys + +left, top, right, bottom = map(int, sys.argv[1].replace(" ", "").split(",")) +width = right - left +height = bottom - top +if width < 700 or height < 520: + raise SystemExit("display too small for native scroll-capture smoke") + +start = (left + width * 30 // 100, top + height * 30 // 100) +end = (left + width * 70 // 100, top + height * 70 // 100) +print(f"{start[0]},{start[1]};{end[0]},{end[1]}") +PY + )" +fi +if [[ -z "${SCROLL_POINT:-}" ]]; then + SCROLL_POINT="$( + python3 - "$DRAG_POINTS" <<'PY' +import sys + +start_raw, end_raw = sys.argv[1].split(";") +x1, y1 = map(int, start_raw.split(",")) +x2, y2 = map(int, end_raw.split(",")) +print(f"{(x1 + x2) // 2},{(y1 + y2) // 2}") +PY + )" +fi +BASE_HEIGHT="$( + python3 - "$DRAG_POINTS" <<'PY' +import sys + +start_raw, end_raw = sys.argv[1].split(";") +_, y1 = map(int, start_raw.split(",")) +_, y2 = map(int, end_raw.split(",")) +print(abs(y2 - y1)) +PY +)" + +smoke_log "display bounds: $DISPLAY_BOUNDS" +smoke_log "drag points: $DRAG_POINTS scroll_point=$SCROLL_POINT base_height=$BASE_HEIGHT" +case "$SCROLL_START_METHOD" in + keyboard|toolbar) + ;; + *) + echo "[smoke] FAIL unknown SCROLL_START_METHOD=$SCROLL_START_METHOD" >&2 + exit 2 + ;; +esac +configure_preferences + +RSNAP_NATIVE_HOST_FORCE_REBUILD="${RSNAP_NATIVE_HOST_FORCE_REBUILD:-1}" \ + APP_POST_VERIFY_SETTLE_S="$APP_POST_VERIFY_SETTLE_S" \ + "$ROOT_DIR/scripts/build_and_run.sh" verify +start_scroll_background + +case_started_epoch="$(date +%s)" +press_capture_hotkey +sleep "$OVERLAY_SETTLE_S" +live_hud_focus_rsnap_overlay + +PATH_POINTS="$DRAG_POINTS" \ + PATH_MODE="drag-region" \ + PATH_DRIVER="${PATH_DRIVER:-event}" \ + PATH_DURATION_MS="${DRAG_DURATION_MS:-280}" \ + PATH_RATE_HZ="${PATH_RATE_HZ:-120}" \ + PATH_HOLD_BEFORE_RELEASE_MS="${DRAG_HOLD_BEFORE_RELEASE_MS:-180}" \ + live_hud_run_mouse_path +sleep "$POST_FREEZE_SETTLE_S" +live_hud_focus_rsnap_overlay +case "$SCROLL_START_METHOD" in + keyboard) + press_plain_s + ;; + toolbar) + click_scroll_toolbar_icon + ;; +esac +sleep 0.25 + +case "$SCROLL_DRIVER" in + notification) + SCROLL_COUNT="$SCROLL_COUNT" \ + SCROLL_DELTA_Y="$SCROLL_DELTA_Y" \ + SCROLL_INTERVAL_MS="$SCROLL_INTERVAL_MS" \ + swift "$SCRIPT_DIR/lib/scroll-background-command.swift" + ;; + wheel) + SCROLL_POINT="$SCROLL_POINT" \ + SCROLL_COUNT="$SCROLL_COUNT" \ + SCROLL_DELTA_Y="$SCROLL_DELTA_Y" \ + SCROLL_INTERVAL_MS="$SCROLL_INTERVAL_MS" \ + swift "$SCRIPT_DIR/lib/scroll-wheel-burst.swift" + ;; + *) + echo "[smoke] FAIL unknown SCROLL_DRIVER=$SCROLL_DRIVER" >&2 + exit 2 + ;; +esac +if [[ "$SCROLL_DRIVER" == "notification" || "$SCROLL_DRIVER" == "wheel" ]]; then + assert_scroll_background_moved +fi +sleep "$POST_SCROLL_SETTLE_S" + +live_hud_focus_rsnap_overlay +osascript -e 'set the clipboard to ""' >/dev/null +press_space +sleep "$POST_COPY_SETTLE_S" +pasteboard_ok=0 +pasteboard_info="" +if pasteboard_info="$( + PASTEBOARD_WAIT_MS="${PASTEBOARD_WAIT_MS:-5000}" \ + swift "$SCRIPT_DIR/lib/pasteboard-image-info.swift" 2>&1 +)"; then + pasteboard_ok=1 + smoke_log "pasteboard $pasteboard_info" +else + smoke_log "pasteboard unavailable: $pasteboard_info" +fi + +telemetry_last="$(( $(date +%s) - case_started_epoch + 10 ))s" +out_dir="$(RSNAP_TELEMETRY_LAST="$telemetry_last" "$ROOT_DIR/scripts/telemetry/native-host.sh" collect)" +smoke_log "telemetry: $out_dir" +parse_telemetry "$out_dir/native-host.oslog" +if [[ "$pasteboard_ok" != "1" ]]; then + echo "[smoke] FAIL no scroll capture image was copied to the pasteboard" >&2 + exit 1 +fi +smoke_log "native scroll-capture smoke passed" diff --git a/scripts/smoke/native-visual-contract-macos.sh b/scripts/smoke/native-visual-contract-macos.sh index 25c69aa5..812cf426 100755 --- a/scripts/smoke/native-visual-contract-macos.sh +++ b/scripts/smoke/native-visual-contract-macos.sh @@ -62,6 +62,9 @@ case "${1:-}" in esac live_hud_self_check +live_hud_start_awake_assertion +live_hud_assert_interactive_session +live_hud_assert_shareable_display ROOT_DIR="$(live_hud_repo_root)" live_hud_init_environment "$ROOT_DIR" @@ -94,6 +97,8 @@ PREF_SNAPSHOT_EXISTS=0 VISUAL_BACKGROUND_PID="" restore_preferences() { + live_hud_stop_awake_assertion + live_hud_cancel_capture_if_present if [[ -n "$VISUAL_BACKGROUND_PID" ]]; then kill "$VISUAL_BACKGROUND_PID" >/dev/null 2>&1 || true fi @@ -113,11 +118,7 @@ fi trap restore_preferences EXIT press_capture_hotkey() { - osascript <<'APPLESCRIPT' >/dev/null -tell application "System Events" - key code 7 using option down -end tell -APPLESCRIPT + live_hud_start_new_capture } open_capture_overlay() { @@ -301,7 +302,7 @@ configure_case_preferences() { run_visual_case() { local case_name="$1" - local case_tmp_dir drag_screenshot_path drag_screenshot_paths screenshot_path background_ready_path case_started_epoch out_dir + local case_tmp_dir drag_screenshot_path drag_screenshot_paths frozen_screenshot_paths screenshot_path background_ready_path case_started_epoch out_dir local mask_probe_path local cursor_helper_bin mask_probe_bin @@ -313,6 +314,7 @@ run_visual_case() { swiftc "$SCRIPT_DIR/lib/mask-probe-capture.swift" -o "$mask_probe_bin" smoke_log "compiled visual smoke helpers" drag_screenshot_paths="" + frozen_screenshot_paths="" mask_probe_paths="" drag_screenshot_path="$case_tmp_dir/dragging-1.png" screenshot_path="$case_tmp_dir/frozen-1.png" @@ -517,6 +519,7 @@ PY fi smoke_log "drag $drag_index captured frozen screenshot" drag_screenshot_paths="${drag_screenshot_paths:+$drag_screenshot_paths:}$drag_screenshot_path" + frozen_screenshot_paths="${frozen_screenshot_paths:+$frozen_screenshot_paths:}$screenshot_path" mask_probe_paths="${mask_probe_paths:+$mask_probe_paths:}$mask_probe_path" close_capture_overlay smoke_log "drag $drag_index closed overlay" @@ -591,6 +594,7 @@ PY EXPECTED_MAX_FROZEN_TRANSFORM_COMMITS="$max_transform_commits" \ EXPECTED_FREEZE_EDITABILITY="$expected_editability" \ VISUAL_DRAG_SCREENSHOT_PATH="$drag_screenshot_paths" \ + VISUAL_FROZEN_SCREENSHOT_PATHS="$frozen_screenshot_paths" \ VISUAL_DISPLAY_BOUNDS="$DISPLAY_BOUNDS" \ VISUAL_DRAG_POINTS="$DRAG_POINTS" \ VISUAL_SCREENSHOT_PATH="$summary_screenshot_path" \ From 036be05a4fb1b3ce4852c1262b143931bbea86ce Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 11 May 2026 22:47:38 +0800 Subject: [PATCH 2/3] Fix scroll capture stitching validation --- ...ptureSessionController+ScrollCapture.swift | 382 ++++++++++---- .../CaptureSessionController.swift | 6 +- .../FrozenCaptureModels.swift | 4 + packages/rsnap-overlay/src/lib.rs | 15 +- packages/rsnap-overlay/src/scroll_capture.rs | 239 +++++++-- .../src/scroll_capture/downward_resolution.rs | 55 +- .../src/scroll_capture/support.rs | 400 +++++++++++++- .../rsnap-overlay/src/scroll_capture/tests.rs | 495 +++++++++++++++++- scripts/smoke/lib/pasteboard-image-info.swift | 8 + .../smoke/lib/scroll-background-window.swift | 129 ++++- .../smoke/lib/scroll-export-continuity.swift | 157 ++++++ scripts/smoke/native-scroll-capture-macos.sh | 97 +++- 12 files changed, 1771 insertions(+), 216 deletions(-) create mode 100644 scripts/smoke/lib/scroll-export-continuity.swift diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift index d70cdce6..dfb2be28 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift @@ -42,7 +42,6 @@ private struct NativeScrollCaptureSampleFrame: Sendable { let source: String let frameSequence: UInt64 let frameAgeMicroseconds: UInt64 - let prefersPairwiseRegistration: Bool } private struct NativeScrollCaptureFallbackRequest: Sendable { @@ -58,13 +57,14 @@ private struct NativeScrollCaptureObservation: Sendable { let errorDescription: String? } -private let nativeScrollCaptureMinimumHintRowsForHintedRegistration = 1 +private let nativeScrollCaptureMinimumNonzeroWheelMotionHintRows = 12.0 private struct NativeScrollCapturePreviewUpdate: @unchecked Sendable { let image: CGImage let exportWidth: Int let exportHeight: Int let result: ScrollObserveResult + let viewportTopYPixels: Int let viewportHeightPixels: Int } @@ -94,8 +94,11 @@ private func writeNativeScrollCaptureDebugDump(_ snapshot: RGBARegionSnapshot, n extension CaptureSessionController { private static let scrollCaptureForwardedEventMarker: Int64 = 0x5253_4E41_5053_4352 - private static let scrollCapturePreciseWheelDeltaLimit = 120.0 + private static let scrollCapturePreciseWheelDeltaLimit = 72.0 + private static let scrollCapturePreciseWheelDeltaMinimum = 36.0 private static let scrollCaptureLineWheelDeltaLimit = 12.0 + private static let scrollCaptureLineWheelDeltaMinimum = 1.0 + private static let scrollCaptureQueuedWheelDeltaLimitMultiplier = 32.0 var scrollCaptureToolbarEnabled: Bool { Self.scrollCaptureEnabled @@ -142,6 +145,10 @@ extension CaptureSessionController { x: viewportPoint.x.clamped(to: state.viewportRect.minX...state.viewportRect.maxX), y: viewportPoint.y.clamped(to: state.viewportRect.minY...state.viewportRect.maxY) ) + if state.controlledScrollInFlight { + queueNativeScrollCaptureWheel(event, at: targetPoint) + return true + } guard nativeScrollCaptureAcceptsManualInput(state: state) else { return true } @@ -166,8 +173,6 @@ extension CaptureSessionController { return true } - scheduleNativeScrollCaptureSample() - return true } @@ -274,15 +279,21 @@ extension CaptureSessionController { return true } - guard var state = scrollCaptureState else { - return false - } - state.lastForwardedWheelUptime = ProcessInfo.processInfo.systemUptime - scrollCaptureState = state - let precise = event.hasPreciseScrollingDeltas - let forwardedDeltaY = Self.forwardedScrollDelta(rawDeltaY, precise: precise) - let postedDeltaY = -forwardedDeltaY + let totalForwardedDeltaY = Self.forwardedScrollQueuedDelta(rawDeltaY, precise: precise) + let forwardedDeltaY = Self.scrollCaptureCommandDelta( + totalForwardedDeltaY, + precise: precise + ) + let overflowDeltaY = totalForwardedDeltaY - forwardedDeltaY + if overflowDeltaY != 0 { + queueNativeScrollCaptureForwardedDelta( + overflowDeltaY, + precise: precise, + at: targetPoint, + source: "overflow" + ) + } if nativeScrollCaptureShouldLogWheelTelemetry( \.lastWheelForwardedTelemetryUptime ) { @@ -290,9 +301,30 @@ extension CaptureSessionController { "capture.scroll_wheel_forwarded", captureID: currentCaptureTelemetryID, detail: - "rawDeltaX=\(Int(rawDeltaX.rounded())),rawDeltaY=\(Int(rawDeltaY.rounded())),forwardedDeltaY=\(Int(forwardedDeltaY.rounded())),postedDeltaY=\(Int(postedDeltaY.rounded())),precise=\(precise)" + "rawDeltaX=\(Int(rawDeltaX.rounded())),rawDeltaY=\(Int(rawDeltaY.rounded())),totalForwardedDeltaY=\(Int(totalForwardedDeltaY.rounded())),forwardedDeltaY=\(Int(forwardedDeltaY.rounded())),queuedOverflowDeltaY=\(Int(overflowDeltaY.rounded())),postedDeltaY=\(Int((-forwardedDeltaY).rounded())),precise=\(precise)" ) } + return postNativeScrollCaptureForwardedDelta( + forwardedDeltaY, + precise: precise, + at: targetPoint + ) + } + + func postNativeScrollCaptureForwardedDelta( + _ forwardedDeltaY: Double, + precise: Bool, + at targetPoint: CGPoint + ) -> Bool { + let postedDeltaY = -forwardedDeltaY + guard postedDeltaY != 0, var state = scrollCaptureState else { + return false + } + + state.lastForwardedWheelUptime = ProcessInfo.processInfo.systemUptime + state.controlledScrollInFlight = true + scrollCaptureState = state + let postWheelEvent = { Self.postScrollWheelEvent( deltaX: 0, @@ -305,19 +337,93 @@ extension CaptureSessionController { overlayController?.withAllMousePassthrough( duration: Self.scrollCaptureForwardingPassthrough ) { - DispatchQueue.main.async { - _ = postWheelEvent() - } - return true + postWheelEvent() } ?? postWheelEvent() if posted { recordNativeScrollCaptureMotionHint(deltaY: abs(postedDeltaY)) - scheduleNativeScrollCaptureSample() + scheduleNativeScrollCaptureSample( + delay: Self.scrollCaptureControlledScrollSettleDelay + ) + } else if var latestState = scrollCaptureState { + latestState.controlledScrollInFlight = false + scrollCaptureState = latestState } return posted } + func queueNativeScrollCaptureWheel(_ event: NSEvent, at targetPoint: CGPoint) { + let rawDeltaY = Double(event.scrollingDeltaY) + guard rawDeltaY != 0 else { + return + } + let precise = event.hasPreciseScrollingDeltas + let forwardedDeltaY = Self.forwardedScrollQueuedDelta(rawDeltaY, precise: precise) + queueNativeScrollCaptureForwardedDelta( + forwardedDeltaY, + precise: precise, + at: targetPoint, + source: "in_flight" + ) + } + + func queueNativeScrollCaptureForwardedDelta( + _ forwardedDeltaY: Double, + precise: Bool, + at targetPoint: CGPoint, + source: String + ) { + guard forwardedDeltaY != 0, var state = scrollCaptureState else { + return + } + let limit = Self.scrollCaptureQueuedWheelDeltaLimit(precise: precise) + + state.queuedForwardedWheelDeltaY = (state.queuedForwardedWheelDeltaY + forwardedDeltaY) + .clamped(to: -limit...limit) + state.queuedForwardedWheelPrecise = precise + state.queuedForwardedWheelTargetPoint = targetPoint + scrollCaptureState = state + if nativeScrollCaptureShouldLogWheelTelemetry( + \.lastWheelInterceptTelemetryUptime + ) { + NativeHostTelemetry.captureEvent( + "capture.scroll_wheel_queued", + captureID: currentCaptureTelemetryID, + detail: + "source=\(source),forwardedDeltaY=\(Int(forwardedDeltaY.rounded())),queuedDeltaY=\(Int(state.queuedForwardedWheelDeltaY.rounded())),precise=\(precise)" + ) + } + } + + func drainNativeScrollCaptureQueuedWheelIfNeeded() { + guard var state = scrollCaptureState, + state.controlledScrollInFlight == false, + state.queuedForwardedWheelDeltaY != 0 + else { + return + } + + let precise = state.queuedForwardedWheelPrecise + let limit = Self.scrollCaptureCommandWheelDeltaLimit(precise: precise) + let forwardedDeltaY = state.queuedForwardedWheelDeltaY.clamped(to: -limit...limit) + let targetPoint = + state.queuedForwardedWheelTargetPoint + ?? CGPoint(x: state.viewportRect.midX, y: state.viewportRect.midY) + + state.queuedForwardedWheelDeltaY -= forwardedDeltaY + if abs(state.queuedForwardedWheelDeltaY) < 1 { + state.queuedForwardedWheelDeltaY = 0 + state.queuedForwardedWheelTargetPoint = nil + } + scrollCaptureState = state + + _ = postNativeScrollCaptureForwardedDelta( + forwardedDeltaY, + precise: precise, + at: targetPoint + ) + } + func recordNativeScrollCaptureMotionHint(deltaY: Double, multiplier: Double = 1) { guard deltaY > 0, var state = scrollCaptureState else { return @@ -325,7 +431,13 @@ extension CaptureSessionController { let viewportHeightRows = state.viewportRect.height * CGFloat(state.viewportPixelsPerPointY) let maxHintRows = max(Double(viewportHeightRows) * 0.85, 1) - let hintRows = deltaY * state.viewportPixelsPerPointY * max(multiplier, 0) + let scaledHintRows = deltaY * state.viewportPixelsPerPointY * max(multiplier, 0) + let hintRows = + if scaledHintRows > 0 { + max(scaledHintRows, nativeScrollCaptureMinimumNonzeroWheelMotionHintRows) + } else { + 0.0 + } state.pendingDownwardMotionHintRows = min( state.pendingDownwardMotionHintRows + hintRows, maxHintRows @@ -335,7 +447,8 @@ extension CaptureSessionController { func scheduleNativeScrollCaptureSample( extendingWindowBy window: TimeInterval = - CaptureSessionController.scrollCaptureInputSampleWindow + CaptureSessionController.scrollCaptureInputSampleWindow, + delay: TimeInterval = CaptureSessionController.scrollCaptureSampleInterval ) { guard var state = scrollCaptureState else { return @@ -352,7 +465,7 @@ extension CaptureSessionController { state.sampleLoopScheduled = true scrollCaptureState = state scheduleNativeScrollCaptureToolbarBackdropRefresh() - DispatchQueue.main.asyncAfter(deadline: .now() + Self.scrollCaptureSampleInterval) { + DispatchQueue.main.asyncAfter(deadline: .now() + max(delay, 0)) { [weak self] in self?.observeNativeScrollCaptureFrame() } @@ -428,7 +541,8 @@ extension CaptureSessionController { } ensureFrozenBaseImageFromDisplayIfNeeded(for: selection) - let frozenBaseImage = chromeState.frozenBaseImage ?? frozenBaseImageFromDisplay(for: selection) + let frozenBaseImage = + chromeState.frozenBaseImage ?? frozenBaseImageFromDisplay(for: selection) guard let frozenBaseImage, let frozenBaseSnapshot = NativeHostImageBridge.rgbaSnapshot(from: frozenBaseImage) else { @@ -436,28 +550,9 @@ extension CaptureSessionController { refreshOverlay() return } - let liveBaseImage = CaptureOverlayController.captureImageBelowOverlay( - in: selection, - source: captureSource - ) - let liveBaseSnapshot = liveBaseImage.flatMap { - NativeHostImageBridge.rgbaSnapshot(from: $0) - } - let liveBaseMatchesFrozenSize = - liveBaseSnapshot?.width == frozenBaseSnapshot.width - && liveBaseSnapshot?.height == frozenBaseSnapshot.height - let baseImage: CGImage - let baseSnapshot: RGBARegionSnapshot - let baseSource: String - if let liveBaseImage, let liveBaseSnapshot, liveBaseMatchesFrozenSize { - baseImage = liveBaseImage - baseSnapshot = liveBaseSnapshot - baseSource = "below_overlay_capture_region" - } else { - baseImage = frozenBaseImage - baseSnapshot = frozenBaseSnapshot - baseSource = "frozen_display_region" - } + let baseImage = frozenBaseImage + let baseSnapshot = frozenBaseSnapshot + let baseSource = "frozen_display_region" debugDumpNativeScrollCaptureSnapshot(baseSnapshot, name: "base-\(baseSource)") let stitcher = try RsnapScrollCaptureSession( baseImage: baseSnapshot, @@ -474,30 +569,17 @@ extension CaptureSessionController { scrollCaptureState = initialState installNativeScrollCaptureMonitor() overlayController?.prepareCaptureStreamsNow(trigger: "scroll_capture_start") - chromeState.frozenOverlay.reset() - chromeState.frozenSelectionEditable = false - chromeState.frozenSelectionInteraction = nil - chromeState.frozenSelectionSnapshot = selection - chromeState.captureFrameSource = .scrollCapture - chromeState.captureFrameWindowID = nil - chromeState.frozenDisplayFrame = nil - chromeState.frozenDisplayImage = nil - chromeState.frozenBaseImage = baseImage - chromeState.scrollMinimapPreview = ScrollCaptureMinimapSnapshot( - image: baseImage, - exportSizePixels: CGSize( - width: CGFloat(baseSnapshot.width), - height: CGFloat(baseSnapshot.height) - ), - viewportTopYPixels: 0, - viewportHeightPixels: CGFloat(baseSnapshot.height) + configureNativeScrollCaptureChrome( + baseImage: baseImage, + baseSnapshot: baseSnapshot, + selection: selection ) try setHostStatusMessage("Scroll capture started. Scroll inside the selection.") NativeHostTelemetry.captureEvent( "capture.scroll_capture_started", captureID: currentCaptureTelemetryID, detail: - "width=\(baseSnapshot.width),height=\(baseSnapshot.height),x=\(Int(selection.minX.rounded())),y=\(Int(selection.minY.rounded())),mode=manual_universal,baseSource=\(baseSource),liveBaseMatched=\(liveBaseMatchesFrozenSize)" + "width=\(baseSnapshot.width),height=\(baseSnapshot.height),x=\(Int(selection.minX.rounded())),y=\(Int(selection.minY.rounded())),mode=manual_universal,baseSource=\(baseSource),liveBaseMatched=false" ) NativeHostTelemetry.captureEvent( "capture.scroll_capture_mode", @@ -506,7 +588,6 @@ extension CaptureSessionController { detail: "input=selection_passthrough_global_monitor,permission=screen_recording,accessibility_required=false" ) - refreshOverlay() overlayController?.focusWindow(at: CGPoint(x: selection.midX, y: selection.midY)) overlayController?.setScrollCaptureMousePassthroughActive(true) NativeHostTelemetry.captureEvent( @@ -515,12 +596,40 @@ extension CaptureSessionController { detail: "input=selection_passthrough_global_monitor,overlay=focused,passthrough=window" ) + DispatchQueue.main.async { [weak self] in + self?.refreshOverlay() + } scheduleNativeScrollCaptureSample( extendingWindowBy: Self.scrollCaptureInitialSampleWindow ) scheduleNativeScrollCaptureToolbarBackdropRefresh() } + private func configureNativeScrollCaptureChrome( + baseImage: CGImage, + baseSnapshot: RGBARegionSnapshot, + selection: CGRect + ) { + chromeState.frozenOverlay.reset() + chromeState.frozenSelectionEditable = false + chromeState.frozenSelectionInteraction = nil + chromeState.frozenSelectionSnapshot = selection + chromeState.captureFrameSource = .scrollCapture + chromeState.captureFrameWindowID = nil + chromeState.frozenDisplayFrame = nil + chromeState.frozenDisplayImage = nil + chromeState.frozenBaseImage = baseImage + chromeState.scrollMinimapPreview = ScrollCaptureMinimapSnapshot( + image: baseImage, + exportSizePixels: CGSize( + width: CGFloat(baseSnapshot.width), + height: CGFloat(baseSnapshot.height) + ), + viewportTopYPixels: 0, + viewportHeightPixels: CGFloat(baseSnapshot.height) + ) + } + func observeNativeScrollCaptureFrame() { guard var state = scrollCaptureState else { return @@ -532,6 +641,14 @@ extension CaptureSessionController { return } state.sampleLoopScheduled = false + if let settleDelay = nativeScrollCaptureControlledSettleDelayRemaining(state: state) { + scrollCaptureState = state + scheduleNativeScrollCaptureSample( + extendingWindowBy: settleDelay + Self.scrollCaptureSampleInterval, + delay: settleDelay + ) + return + } state.sampleProcessing = true state.sampleSequence &+= 1 let sampleSequence = state.sampleSequence @@ -552,7 +669,8 @@ extension CaptureSessionController { let sampledFrames = nativeScrollCaptureSampleFrames( in: state.viewportRect, - afterFrameSequence: state.lastStreamFrameSequence + afterFrameSequence: state.lastStreamFrameSequence, + maximumFrameAgeMicroseconds: nativeScrollCaptureMaximumStreamFrameAge(state: state) ) let fallbackRequest = sampledFrames.isEmpty @@ -624,8 +742,7 @@ extension CaptureSessionController { region: snapshot, source: "below_overlay_capture_region", frameSequence: fallbackRequest.frameSequence, - frameAgeMicroseconds: 0, - prefersPairwiseRegistration: true + frameAgeMicroseconds: 0 )) } let batch = Self.nativeScrollCaptureObservationBatch( @@ -683,22 +800,12 @@ extension CaptureSessionController { stitcher: RsnapScrollCaptureSession, motionRowsHint: Int? ) -> NativeScrollCaptureObservation { - let usesHintedRegistration = nativeScrollCaptureUsesHintedRegistration( - for: sampledFrame, - motionRowsHint: motionRowsHint - ) - let registrationStrategy = usesHintedRegistration ? "hinted_motion" : "pairwise" + let registrationStrategy = "pairwise" do { - let result = - if usesHintedRegistration, let motionRowsHint { - try stitcher.observeDownwardFrame( - sampledFrame.region, - motionRowsHint: motionRowsHint, - allowBurstSearch: true - ) - } else { - try stitcher.observeDownwardFrame(sampledFrame.region) - } + let result = try stitcher.observeDownwardFrame( + sampledFrame.region, + motionRowsHint: motionRowsHint + ) return NativeScrollCaptureObservation( sampledFrame: sampledFrame, registrationStrategy: registrationStrategy, @@ -715,19 +822,6 @@ extension CaptureSessionController { } } - nonisolated private static func nativeScrollCaptureUsesHintedRegistration( - for sampledFrame: NativeScrollCaptureSampleFrame, - motionRowsHint: Int? - ) -> Bool { - guard sampledFrame.prefersPairwiseRegistration == false, - let motionRowsHint, - motionRowsHint >= nativeScrollCaptureMinimumHintRowsForHintedRegistration - else { - return false - } - return true - } - nonisolated private static func nativeScrollCapturePreviewUpdate( stitcher: RsnapScrollCaptureSession, candidate: NativeScrollCaptureObservation?, @@ -742,15 +836,21 @@ extension CaptureSessionController { } let previewStartedAt = ProcessInfo.processInfo.systemUptime do { - if let export = try stitcher.exportImage(), - let exportImage = NativeHostImageBridge.cgImage(from: export) - { + if let export = try stitcher.exportImage() { + guard let exportImage = NativeHostImageBridge.cgImage(from: export) else { + return ( + nil, + "scroll preview export returned no image", + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } return ( NativeScrollCapturePreviewUpdate( image: exportImage, exportWidth: export.width, exportHeight: export.height, result: result, + viewportTopYPixels: result.currentViewportTopY, viewportHeightPixels: candidate.sampledFrame.region.height ), nil, @@ -787,6 +887,7 @@ extension CaptureSessionController { state.sampleProcessing = false scrollCaptureState = state defer { + completeNativeScrollCaptureCommandIfNeeded() scheduleNativeScrollCaptureSampleIfNeeded() } guard batch.observations.isEmpty == false else { @@ -815,6 +916,8 @@ extension CaptureSessionController { if result.outcome == .committed { latestState.committedSampleCount &+= 1 latestState.pendingDownwardMotionHintRows = 0 + } else if result.outcome == .noChange, latestState.controlledScrollInFlight { + latestState.pendingDownwardMotionHintRows = 0 } else if result.outcome == .unsupportedDirection { latestState.pendingDownwardMotionHintRows = 0 } @@ -877,12 +980,36 @@ extension CaptureSessionController { writeNativeScrollCaptureDebugDump(snapshot, name: name) } - func nativeScrollCaptureAcceptsManualInput(state _: NativeScrollCaptureState) -> Bool { - true + func nativeScrollCaptureAcceptsManualInput(state: NativeScrollCaptureState) -> Bool { + state.controlledScrollInFlight == false + } + + func nativeScrollCaptureControlledSettleDelayRemaining( + state: NativeScrollCaptureState + ) -> TimeInterval? { + guard state.controlledScrollInFlight, state.lastForwardedWheelUptime > 0 else { + return nil + } + let elapsed = ProcessInfo.processInfo.systemUptime - state.lastForwardedWheelUptime + let remaining = Self.scrollCaptureControlledScrollSettleDelay - elapsed + return remaining > 0 ? remaining : nil + } + + func completeNativeScrollCaptureCommandIfNeeded() { + guard var state = scrollCaptureState, state.controlledScrollInFlight else { + drainNativeScrollCaptureQueuedWheelIfNeeded() + return + } + + state.controlledScrollInFlight = false + scrollCaptureState = state + drainNativeScrollCaptureQueuedWheelIfNeeded() } func nativeScrollCaptureShouldKeepSampling(state: NativeScrollCaptureState) -> Bool { ProcessInfo.processInfo.systemUptime < state.sampleUntilUptime + || state.controlledScrollInFlight + || state.queuedForwardedWheelDeltaY != 0 } func recordNativeScrollCaptureMissingSample( @@ -909,7 +1036,8 @@ extension CaptureSessionController { private func nativeScrollCaptureSampleFrames( in rect: CGRect, - afterFrameSequence: UInt64 + afterFrameSequence: UInt64, + maximumFrameAgeMicroseconds: UInt64? ) -> [NativeScrollCaptureSampleFrame] { var frames: [NativeScrollCaptureSampleFrame] = [] var nextAfterFrameSequence = afterFrameSequence @@ -924,19 +1052,33 @@ extension CaptureSessionController { else { break } + if let maximumFrameAgeMicroseconds, + frame.frameAgeMicroseconds > maximumFrameAgeMicroseconds + { + nextAfterFrameSequence = frame.frameSequence + continue + } frames.append( NativeScrollCaptureSampleFrame( region: frame.region, source: "ordered_live_stream_region", frameSequence: frame.frameSequence, - frameAgeMicroseconds: frame.frameAgeMicroseconds, - prefersPairwiseRegistration: false + frameAgeMicroseconds: frame.frameAgeMicroseconds )) nextAfterFrameSequence = frame.frameSequence } return frames } + private func nativeScrollCaptureMaximumStreamFrameAge( + state: NativeScrollCaptureState + ) -> UInt64? { + guard nativeScrollCaptureFallbackReadyForInput(state: state) else { + return nil + } + return UInt64(Self.scrollCaptureInputLiveFrameMaxAge * 1_000_000) + } + private func nativeScrollCaptureFallbackAllowed(at uptime: TimeInterval) -> Bool { guard var state = scrollCaptureState else { return false @@ -1011,7 +1153,7 @@ extension CaptureSessionController { width: CGFloat(preview.exportWidth), height: CGFloat(preview.exportHeight) ), - viewportTopYPixels: CGFloat(preview.result.currentViewportTopY), + viewportTopYPixels: CGFloat(preview.viewportTopYPixels), viewportHeightPixels: CGFloat(preview.viewportHeightPixels) ) @@ -1061,8 +1203,36 @@ extension CaptureSessionController { return true } - private static func forwardedScrollDelta(_ delta: Double, precise: Bool) -> Double { - let limit = precise ? scrollCapturePreciseWheelDeltaLimit : scrollCaptureLineWheelDeltaLimit + private static func scrollCaptureCommandWheelDeltaLimit(precise: Bool) -> Double { + precise ? scrollCapturePreciseWheelDeltaLimit : scrollCaptureLineWheelDeltaLimit + } + + private static func scrollCaptureQueuedWheelDeltaLimit(precise: Bool) -> Double { + scrollCaptureCommandWheelDeltaLimit(precise: precise) + * scrollCaptureQueuedWheelDeltaLimitMultiplier + } + + private static func scrollCaptureCommandDelta(_ delta: Double, precise: Bool) -> Double { + let commandDelta = forwardedScrollDelta( + delta, + limit: scrollCaptureCommandWheelDeltaLimit(precise: precise) + ) + let minimum = + precise ? scrollCapturePreciseWheelDeltaMinimum : scrollCaptureLineWheelDeltaMinimum + guard commandDelta != 0, abs(commandDelta) < minimum else { + return commandDelta + } + return commandDelta < 0 ? -minimum : minimum + } + + private static func forwardedScrollQueuedDelta(_ delta: Double, precise: Bool) -> Double { + forwardedScrollDelta( + delta, + limit: scrollCaptureQueuedWheelDeltaLimit(precise: precise) + ) + } + + private static func forwardedScrollDelta(_ delta: Double, limit: Double) -> Double { let clamped = delta.clamped(to: -limit...limit) let rounded = clamped.rounded() if abs(rounded) >= 1 { diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift index 6f68b057..52cbde10 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController.swift @@ -31,11 +31,13 @@ final class CaptureSessionController: NSObject { static let displayFirstFrameWait: TimeInterval = 0.025 static let coldSelfCaptureRecoveryWait: TimeInterval = 3.5 static let scrollCaptureEnabled = true - static let scrollCaptureForwardingPassthrough: TimeInterval = 0.08 + static let scrollCaptureForwardingPassthrough: TimeInterval = 0.012 + static let scrollCaptureControlledScrollSettleDelay: TimeInterval = 0.18 + static let scrollCaptureInputLiveFrameMaxAge: TimeInterval = 0.18 static let scrollCaptureSampleInterval: TimeInterval = 1.0 / 30.0 static let scrollCaptureMaxFramesPerSample = 3 static let scrollCaptureInitialSampleWindow: TimeInterval = 0.35 - static let scrollCaptureInputSampleWindow: TimeInterval = 0.85 + static let scrollCaptureInputSampleWindow: TimeInterval = 1.8 static let scrollCaptureFallbackCaptureInterval: TimeInterval = 0.08 static let scrollCapturePreviewRefreshInterval: TimeInterval = 0.18 static let scrollCaptureToolbarBackdropRefreshInterval: TimeInterval = 1.0 / 120.0 diff --git a/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift b/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift index 6c06521f..fd193388 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/FrozenCaptureModels.swift @@ -560,6 +560,10 @@ struct NativeScrollCaptureState { var lastStreamFrameSequence: UInt64 = 0 var lastMissingSampleStatusUptime: TimeInterval = 0 var lastForwardedWheelUptime: TimeInterval = 0 + var controlledScrollInFlight = false + var queuedForwardedWheelDeltaY: Double = 0 + var queuedForwardedWheelPrecise = true + var queuedForwardedWheelTargetPoint: CGPoint? var lastFallbackCaptureUptime: TimeInterval = 0 var lastPreviewRefreshUptime: TimeInterval = 0 var lastWheelInterceptTelemetryUptime: TimeInterval = 0 diff --git a/packages/rsnap-overlay/src/lib.rs b/packages/rsnap-overlay/src/lib.rs index 0b60db13..63047217 100644 --- a/packages/rsnap-overlay/src/lib.rs +++ b/packages/rsnap-overlay/src/lib.rs @@ -91,27 +91,20 @@ pub mod scroll_stitching { .map(scroll_stitch_observe_outcome_from) } - /// Observes a discrete native screenshot with an optional downward motion hint. - /// - /// Native macOS auto-scroll knows the commanded AX scroll delta before the next - /// stable frame arrives. Feeding that as a hint lets the stitcher resolve bursty - /// or repeated-content frames without relaxing rewind safety for unhinted samples. + /// Observes a discrete native screenshot with pairwise registration and an + /// optional downward motion hint for committed-frontier catch-up. pub fn observe_downward_rgba_with_motion_hint( &mut self, width: u32, height: u32, rgba: &[u8], motion_rows_hint: Option, - allow_burst_search: bool, + _allow_burst_search: bool, ) -> Result { let frame = rgba_image_from_bytes(width, height, rgba)?; self.inner - .observe_downward_sample_with_motion_hint_and_burst( - frame, - motion_rows_hint, - allow_burst_search, - ) + .observe_worker_pairwise_vision_frame_with_motion_hint(frame, motion_rows_hint) .map(scroll_stitch_observe_outcome_from) } diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 2f7728cf..641c1ad7 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -18,6 +18,7 @@ use self::support::detect_vertical_overlap; pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 24; pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS: u32 = 12; pub(crate) const UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS: u32 = 8; +pub(crate) const TRANSIENT_BURST_UNDERCONSUMED_HINT_MIN_ROWS: u32 = 48; const FINGERPRINT_GRID_COLUMNS: u32 = 12; const FINGERPRINT_GRID_ROWS: u32 = 16; @@ -45,6 +46,7 @@ const FALLBACK_DOWNWARD_GROWTH_MAX_ROWS: u32 = 16; const TRANSIENT_MOTION_HINT_MAX_MULTIPLIER: u32 = 3; const TRANSIENT_MOTION_HINT_MIN_CAP_ROWS: u32 = 12; const WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS: u32 = 24; +const WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS: u32 = 24; const DIRECTION_WARNING_MARGIN_X100: u32 = 90; const RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100: u32 = 320; const INFORMATIVE_SPAN_ROW_SAMPLES: u32 = 24; @@ -184,8 +186,8 @@ impl Default for OverlapSearchConfig { fn default() -> Self { Self { min_overlap_rows: 24, - max_column_samples: 32, - max_row_samples: 16, + max_column_samples: 160, + max_row_samples: 64, max_mean_abs_diff_x100: 850, } } @@ -386,6 +388,17 @@ impl ScrollSession { )); } + if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { + self.worker_pairwise_requires_committed_reacquire = false; + + return self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, + motion_rows, + Some(motion_rows), + ); + } + return Ok(self.block_worker_pairwise_until_committed_reacquire(frame, fingerprint)); } if frame == previous_worker_frame { @@ -398,10 +411,45 @@ impl ScrollSession { return Ok(self.observe_worker_pairwise_no_change(frame, fingerprint, reason)); } - let Some(matched) = self::support::classify_vision_downward_sample_motion_against( + let vision_match = self::support::classify_vision_downward_sample_motion_against( &previous_worker_frame, &frame, - ) else { + ); + let (matched, corroborated_shift_rows) = if let Some(matched) = vision_match { + ( + matched, + self::support::trusted_pairwise_downward_shift_rows_near_motion( + &previous_worker_frame, + &frame, + matched.motion_rows, + WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, + ), + ) + } else if let Some(matched) = + self::support::trusted_pairwise_downward_shift_match(&previous_worker_frame, &frame) + { + let max_pixel_fallback_motion_rows = + previous_worker_frame.height().saturating_div(2).max(1); + + if matched.motion_rows > max_pixel_fallback_motion_rows { + let candidate_viewport_top_y = self + .current_viewport_top_y + .saturating_add(i32::try_from(matched.motion_rows).unwrap_or_default()); + let growth_rows = + self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + + return Ok(self.block_worker_pairwise_growth( + frame, + fingerprint, + matched.motion_rows, + candidate_viewport_top_y, + growth_rows, + "worker_pairwise_pixel_overlap_exceeded_fallback_budget", + )); + } + + (matched, Some(matched.motion_rows)) + } else { if let Some(upward_motion_rows) = self::support::trusted_pairwise_upward_shift_rows(&previous_worker_frame, &frame) { @@ -411,20 +459,21 @@ impl ScrollSession { upward_motion_rows, )); } + if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { + return self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, + motion_rows, + Some(motion_rows), + ); + } return Ok(self.observe_worker_pairwise_no_change( frame, fingerprint, - "worker_pairwise_vision_no_downward_offset", + "worker_pairwise_no_downward_offset", )); }; - let corroborated_shift_rows = - self::support::trusted_pairwise_downward_shift_rows_near_motion( - &previous_worker_frame, - &frame, - matched.motion_rows, - WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, - ); self.observe_resolved_worker_pairwise_downward_motion( frame, @@ -434,6 +483,52 @@ impl ScrollSession { ) } + pub(crate) fn observe_worker_pairwise_vision_frame_with_motion_hint( + &mut self, + frame: RgbaImage, + motion_rows_hint: Option, + ) -> color_eyre::eyre::Result { + let previous_hint = self.transient_motion_rows_hint; + + self.transient_motion_rows_hint = motion_rows_hint; + + self.record_last_sample_eval_context(); + + let result = self.observe_worker_pairwise_vision_frame(frame); + + self.transient_motion_rows_hint = previous_hint; + + result + } + + fn worker_pairwise_committed_catchup_motion_rows(&self, frame: &RgbaImage) -> Option { + if frame == &self.last_committed_frame { + return None; + } + + let hinted_match = self.normalized_transient_motion_rows_hint().and_then(|hint| { + let tolerance = (hint / 2).clamp( + WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, + INITIAL_DOWNWARD_MAX_MOTION_ROWS, + ); + + self::support::trusted_pairwise_downward_shift_rows_near_motion( + &self.last_committed_frame, + frame, + hint, + tolerance, + ) + }); + let fallback_match = || { + self::support::trusted_pairwise_downward_shift_match(&self.last_committed_frame, frame) + .map(|matched| matched.motion_rows) + }; + + hinted_match + .or_else(fallback_match) + .filter(|motion_rows| *motion_rows >= WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS) + } + fn observe_resolved_worker_pairwise_downward_motion( &mut self, frame: RgbaImage, @@ -547,9 +642,7 @@ impl ScrollSession { fingerprint: Vec, reason: &'static str, ) -> ScrollObserveOutcome { - if reason == "worker_pairwise_vision_no_downward_offset" - && frame != self.last_committed_frame - { + if reason == "worker_pairwise_no_downward_offset" && frame != self.last_committed_frame { self.record_last_sample(&frame, fingerprint); self.clear_preview_only_downward_recovery_carryover(); @@ -1622,7 +1715,6 @@ impl ScrollSession { self.observed_viewport_top_y.min(frontier_top_y.saturating_sub(1)); } - #[allow(clippy::too_many_lines)] fn observe_downward_motion( &mut self, frame: RgbaImage, @@ -1675,34 +1767,68 @@ impl ScrollSession { }, }; + if let Some(outcome) = + self.block_invalid_downward_candidate(&frame, motion_rows, candidate, preview_changed)? + { + return Ok(outcome); + } + + self.observe_downward_growth_to_viewport( + frame, + candidate.viewport_top_y, + preview_changed, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: candidate.motion_rows, + }), + candidate.source.decision_source(), + ) + } + + fn block_invalid_downward_candidate( + &mut self, + frame: &RgbaImage, + motion_rows: u32, + candidate: DownwardViewportCandidate, + preview_changed: bool, + ) -> color_eyre::eyre::Result> { + if self.transient_burst_candidate_underconsumes_input_hint(candidate) { + return Ok(Some(self.block_downward_growth_candidate( + frame, + motion_rows, + candidate, + preview_changed, + "visual_motion_underconsumed_input_hint", + )?)); + } if self.should_fail_closed_tiny_observed_recovery_in_burst(candidate) { - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "tiny_observed_recovery_under_transient_burst", - ); + )?)); } if self.should_fail_closed_outsized_observed_recovery_after_one_pixel_preview_local_commit( candidate, ) { - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "outsized_observed_recovery_after_one_pixel_preview_local_commit", - ); + )?)); } if self.should_fail_closed_tiny_preview_only_local_recovery_in_burst(candidate) { - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "tiny_preview_only_local_recovery_under_transient_burst", - ); + )?)); } if self .should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst(candidate) @@ -1710,43 +1836,34 @@ impl ScrollSession { self.pending_extreme_preview_only_local_tail_followup = Some(candidate); self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 1; - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "exactly_corroborated_preview_local_tail_under_extreme_transient_burst", - ); + )?)); } if self.should_fail_closed_preview_only_local_tail_after_unresolved_burst(candidate) { - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "preview_only_local_tail_after_unresolved_transient_burst", - ); + )?)); } if self.should_fail_closed_tiny_committed_keyframe_recovery_in_burst(candidate) { - return self.block_downward_growth_candidate( - &frame, + return Ok(Some(self.block_downward_growth_candidate( + frame, motion_rows, candidate, preview_changed, "tiny_committed_keyframe_recovery_under_transient_burst", - ); + )?)); } - self.observe_downward_growth_to_viewport( - frame, - candidate.viewport_top_y, - preview_changed, - Some(MotionObservation { - direction: ScrollDirection::Down, - motion_rows: candidate.motion_rows, - }), - candidate.source.decision_source(), - ) + Ok(None) } fn handle_missing_downward_viewport_authority( @@ -1795,13 +1912,8 @@ impl ScrollSession { } else { None }; - self.consecutive_transient_burst_missing_downward_candidate_frames = - if self.transient_burst_search_enabled && preview_only_local_viewport_top_y.is_some() { - self.consecutive_transient_burst_missing_downward_candidate_frames.saturating_add(1) - } else { - 0 - }; + self.record_transient_burst_missing_downward_candidate_frame(preview_changed); self.refresh_local_downward_sample(frame); if self.should_refresh_downward_observed_baseline_after_huge_suppressed_jump() { @@ -2513,7 +2625,13 @@ impl ScrollSession { return self .transient_motion_rows_hint .map(|hint| { - frame_max_growth_rows.min(hint.max(INITIAL_DOWNWARD_MAX_MOTION_ROWS)).max(1) + if self.initial_downward_bootstrap_active() + && self.last_motion_rows_hint.is_none() + { + frame_max_growth_rows + } else { + frame_max_growth_rows.min(hint.max(INITIAL_DOWNWARD_MAX_MOTION_ROWS)).max(1) + } }) .unwrap_or(frame_max_growth_rows.clamp(1, INITIAL_DOWNWARD_MAX_MOTION_ROWS)); } @@ -2805,7 +2923,8 @@ impl ScrollSession { match registration { DownwardRegistration::Matched(matched) - if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) => + if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) + && !self.bootstrap_hint_exceeded_match_can_commit(matched) => { (DownwardRegistration::NoMatch, Some("bootstrap_hint_exceeded")) }, @@ -2835,7 +2954,8 @@ impl ScrollSession { match registration { DownwardRegistration::Matched(matched) - if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) => + if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) + && !self.bootstrap_hint_exceeded_match_can_commit(matched) => { (DownwardRegistration::NoMatch, Some("bootstrap_hint_exceeded")) }, @@ -2847,6 +2967,9 @@ impl ScrollSession { let transient = self.normalized_transient_motion_rows_hint(); match (self.last_motion_rows_hint, transient) { + (Some(last), Some(transient)) if self.transient_burst_search_enabled => { + Some(last.max(transient)) + }, (Some(last), Some(_transient)) => Some(last), (Some(last), None) => Some(last), (None, Some(transient)) => Some(transient), @@ -3096,10 +3219,20 @@ impl ScrollSession { self.bootstrap_motion_cap_from_pending_hint().is_some_and(|cap| motion_rows > cap) } + fn bootstrap_hint_exceeded_match_can_commit(&self, matched: DirectionMatch) -> bool { + self.initial_downward_bootstrap_active() + && self.transient_burst_search_enabled + && self.last_motion_rows_hint.is_none() + && matched.mean_abs_diff_x100 <= DIRECTION_WARNING_MARGIN_X100.saturating_mul(4) + } + fn bootstrap_initial_growth_cap_rows(&self) -> Option { if !self.initial_downward_bootstrap_active() || self.last_motion_rows_hint.is_some() { return None; } + if self.transient_burst_search_enabled { + return None; + } self.bootstrap_motion_cap_from_pending_hint() .map(|cap| cap.min(BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS)) diff --git a/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs b/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs index 47da6bae..7486bbb1 100644 --- a/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs +++ b/packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs @@ -17,7 +17,8 @@ use crate::scroll_capture::{ REPEATED_PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS, RangeInclusive, RgbaImage, ScrollDirection, ScrollObserveOutcome, ScrollSession, TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS, - UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS, eyre, + TRANSIENT_BURST_UNDERCONSUMED_HINT_MIN_ROWS, UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS, + eyre, }; impl ScrollSession { @@ -1738,6 +1739,48 @@ impl ScrollSession { growth_rows > max_growth_rows } + pub(super) fn record_transient_burst_missing_downward_candidate_frame( + &mut self, + preview_changed: bool, + ) { + if self.transient_burst_search_enabled && preview_changed { + self.consecutive_transient_burst_missing_downward_candidate_frames = self + .consecutive_transient_burst_missing_downward_candidate_frames + .saturating_add(1); + } else { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + } + } + + pub(super) fn transient_burst_candidate_underconsumes_input_hint( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if !self.transient_burst_search_enabled || self.resume_frontier_top_y.is_some() { + return false; + } + + let Some(hint) = self.normalized_transient_motion_rows_hint() else { + return false; + }; + + if hint < TRANSIENT_BURST_UNDERCONSUMED_HINT_MIN_ROWS { + return false; + } + + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + let visibly_underconsumed = + growth_rows.saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) < hint; + let severely_underconsumed = growth_rows <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + || growth_rows.saturating_mul(3) < hint; + + visibly_underconsumed + && (severely_underconsumed + || self.transient_burst_motion_hint_exceeds_local_authority( + growth_rows.max(candidate.motion_rows), + )) + } + pub(super) fn observe_fallback_downward_growth( &mut self, frame: RgbaImage, @@ -1749,6 +1792,7 @@ impl ScrollSession { match support::select_downward_viewport_candidate(&mut candidates) { DownwardViewportResolution::NoMatch => { + self.record_transient_burst_missing_downward_candidate_frame(preview_changed); self.refresh_preview_only_downward_local_sample( &frame, self.stable_preview_only_downward_local_viewport_top_y(), @@ -1765,6 +1809,15 @@ impl ScrollSession { Ok(support::preview_update_outcome(preview_changed)) }, DownwardViewportResolution::Selected(candidate) => { + if self.transient_burst_candidate_underconsumes_input_hint(candidate) { + return self.block_downward_growth_candidate( + &frame, + candidate.motion_rows, + candidate, + preview_changed, + "visual_motion_underconsumed_input_hint", + ); + } if self.fallback_downward_growth_exceeds_continuity_budget(candidate.viewport_top_y) { self.refresh_preview_only_downward_local_sample( diff --git a/packages/rsnap-overlay/src/scroll_capture/support.rs b/packages/rsnap-overlay/src/scroll_capture/support.rs index b8f15242..63a5cb07 100644 --- a/packages/rsnap-overlay/src/scroll_capture/support.rs +++ b/packages/rsnap-overlay/src/scroll_capture/support.rs @@ -36,6 +36,18 @@ use crate::scroll_capture::{ ScrollObserveOutcome, }; +const MOTION_COVERAGE_MIN_PERCENT: u32 = 20; +const MOTION_COVERAGE_MIN_INFORMATIVE_COLUMNS: u32 = 1; +const MOTION_COVERAGE_STATIC_EDGE_MAX_LEADING_COLUMNS: u32 = 48; +const MOTION_COVERAGE_STATIC_EDGE_MIN_COLUMNS: u32 = 64; +const MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS: usize = 64; +const MOTION_COVERAGE_STATIC_BAND_MIN_PERCENT: u32 = 65; +const MOTION_COVERAGE_STATIC_BAND_STRUCTURE_DIVISOR: u32 = 64; +const MOTION_COVERAGE_STATIC_BAND_MOTION_DIVISOR: u32 = 16; +const MOTION_OVERLAP_MIN_MATCHING_COLUMN_PERCENT: u32 = 80; +const MOTION_OVERLAP_BAD_EDGE_SAMPLE_DIVISOR: usize = 10; +const MOTION_OVERLAP_BAD_EDGE_MIN_SAMPLES: usize = 8; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum OverlapOrientation { PreviousBottomToNextTop, @@ -221,6 +233,13 @@ pub(super) fn trusted_pairwise_downward_shift_rows_near_motion( } } +pub(super) fn trusted_pairwise_downward_shift_match( + previous: &RgbaImage, + current: &RgbaImage, +) -> Option { + trusted_pairwise_shift_match(previous, current, ScrollDirection::Down) +} + pub(super) fn trusted_pairwise_upward_shift_rows( previous: &RgbaImage, current: &RgbaImage, @@ -348,6 +367,12 @@ pub(super) fn collect_overlap_direction_matches( ScrollDirection::Down => OverlapOrientation::PreviousBottomToNextTop, ScrollDirection::Up => OverlapOrientation::PreviousTopToNextBottom, }; + let sample_columns = motion_sample_columns_for_span(previous, next, informative_span, config); + + if sample_columns.is_empty() { + return Vec::new(); + } + let mut matches = Vec::with_capacity(search_end.saturating_sub(search_start) as usize + 1); for motion_rows in search_start..=search_end { @@ -363,7 +388,7 @@ pub(super) fn collect_overlap_direction_matches( motion_rows, config, orientation, - informative_span, + &sample_columns, ); if diff > config.max_mean_abs_diff_x100 { @@ -804,8 +829,8 @@ fn trusted_pairwise_shift_match( fn worker_pairwise_overlap_search_config() -> OverlapSearchConfig { OverlapSearchConfig { min_overlap_rows: 24, - max_column_samples: 96, - max_row_samples: 96, + max_column_samples: 240, + max_row_samples: 128, max_mean_abs_diff_x100: 850, } } @@ -895,8 +920,13 @@ fn detect_vertical_overlap_in_range( ScrollDirection::Down => OverlapOrientation::PreviousBottomToNextTop, ScrollDirection::Up => OverlapOrientation::PreviousTopToNextBottom, }; + let sample_columns = motion_sample_columns_for_span(previous, next, informative_span, config); let mut best = OverlapMatch { rows: 0, matched: false, mean_abs_diff_x100: u32::MAX }; + if sample_columns.is_empty() { + return best; + } + for motion_rows in search_start..=search_end { let overlap_rows = max_overlap.saturating_sub(motion_rows); @@ -910,7 +940,7 @@ fn detect_vertical_overlap_in_range( motion_rows, config, orientation, - informative_span, + &sample_columns, ); if diff > config.max_mean_abs_diff_x100 { @@ -933,9 +963,8 @@ fn motion_mean_abs_diff_x100( motion_rows: u32, config: OverlapSearchConfig, orientation: OverlapOrientation, - informative_span: InformativeSpan, + sample_columns: &[u32], ) -> u32 { - let width = previous.width().min(next.width()); let max_overlap = previous.height().min(next.height()); let overlap_rows = max_overlap.saturating_sub(motion_rows); @@ -943,7 +972,6 @@ fn motion_mean_abs_diff_x100( return u32::MAX; } - let column_samples = width.min(config.max_column_samples).max(1); let row_samples = overlap_rows.min(config.max_row_samples).max(1); let previous_overlap_start_y = previous.height().saturating_sub(overlap_rows); let next_overlap_start_y = next.height().saturating_sub(overlap_rows); @@ -955,12 +983,10 @@ fn motion_mean_abs_diff_x100( OverlapOrientation::PreviousBottomToNextTop => 0, OverlapOrientation::PreviousTopToNextBottom => next_overlap_start_y, }; - let x_start = informative_span.start_x.min(width.saturating_sub(1)); - let x_end = informative_span.end_exclusive_x.min(width).max(x_start + 1); - let effective_width = x_end.saturating_sub(x_start).max(1); - let column_samples = effective_width.min(column_samples).max(1); let mut total_abs_diff = 0_u64; let mut comparisons = 0_u64; + let mut column_abs_diff = vec![0_u64; sample_columns.len()]; + let mut column_comparisons = 0_u64; for row in 0..row_samples { let local_y = evenly_spaced_sample(0, overlap_rows, row, row_samples); @@ -968,45 +994,373 @@ fn motion_mean_abs_diff_x100( previous_start_y.saturating_add(local_y).min(previous.height().saturating_sub(1)); let next_y = next_start_y.saturating_add(local_y).min(next.height().saturating_sub(1)); - for column in 0..column_samples { - let x = evenly_spaced_sample(x_start, x_end, column, column_samples); - let previous_pixel = previous.get_pixel(x, previous_y).0; - let next_pixel = next.get_pixel(x, next_y).0; - - total_abs_diff = total_abs_diff - .saturating_add(u64::from(previous_pixel[0].abs_diff(next_pixel[0]))) + for (column_index, x) in sample_columns.iter().enumerate() { + let previous_pixel = previous.get_pixel(*x, previous_y).0; + let next_pixel = next.get_pixel(*x, next_y).0; + let pixel_abs_diff = u64::from(previous_pixel[0].abs_diff(next_pixel[0])) .saturating_add(u64::from(previous_pixel[1].abs_diff(next_pixel[1]))) .saturating_add(u64::from(previous_pixel[2].abs_diff(next_pixel[2]))); + + total_abs_diff = total_abs_diff.saturating_add(pixel_abs_diff); + column_abs_diff[column_index] = + column_abs_diff[column_index].saturating_add(pixel_abs_diff); comparisons = comparisons.saturating_add(3); } + + column_comparisons = column_comparisons.saturating_add(3); } if comparisons == 0 { return u32::MAX; } + if !motion_overlap_columns_support_span(&column_abs_diff, column_comparisons, config) { + return u32::MAX; + } ((total_abs_diff.saturating_mul(100)) / comparisons) as u32 } +fn motion_overlap_columns_support_span( + column_abs_diff: &[u64], + column_comparisons: u64, + config: OverlapSearchConfig, +) -> bool { + if column_abs_diff.is_empty() || column_comparisons == 0 { + return false; + } + + let bad_column_threshold = config + .max_mean_abs_diff_x100 + .saturating_mul(4) + .max(config.max_mean_abs_diff_x100.saturating_add(1)); + let mut matching_columns = 0_u32; + let mut bad_columns = Vec::with_capacity(column_abs_diff.len()); + + for total in column_abs_diff { + let column_mean_x100 = ((total.saturating_mul(100)) / column_comparisons) as u32; + let column_matches = column_mean_x100 <= bad_column_threshold; + + if column_matches { + matching_columns = matching_columns.saturating_add(1); + } + + bad_columns.push(!column_matches); + } + + let total_columns = column_abs_diff.len() as u32; + let enough_matching_columns = matching_columns.saturating_mul(100) + >= total_columns.saturating_mul(MOTION_OVERLAP_MIN_MATCHING_COLUMN_PERCENT); + let min_bad_edge_columns = (column_abs_diff.len() / MOTION_OVERLAP_BAD_EDGE_SAMPLE_DIVISOR) + .max(MOTION_OVERLAP_BAD_EDGE_MIN_SAMPLES) + .min(column_abs_diff.len()); + + enough_matching_columns + && leading_true_run_len(bad_columns.iter().copied()) < min_bad_edge_columns + && leading_true_run_len(bad_columns.iter().rev().copied()) < min_bad_edge_columns +} + +fn leading_true_run_len(iter: I) -> usize +where + I: IntoIterator, +{ + let mut len = 0_usize; + + for value in iter { + if !value { + break; + } + + len = len.saturating_add(1); + } + + len +} + +fn motion_sample_columns_for_span( + previous: &RgbaImage, + next: &RgbaImage, + informative_span: InformativeSpan, + config: OverlapSearchConfig, +) -> Vec { + let width = previous.width().min(next.width()); + + if width == 0 { + return Vec::new(); + } + + let x_start = informative_span.start_x.min(width.saturating_sub(1)); + let x_end = informative_span.end_exclusive_x.min(width).max(x_start + 1); + let column_samples = width.min(config.max_column_samples).max(1); + + evenly_sampled_columns(x_start, x_end, column_samples) +} + +fn evenly_sampled_columns(x_start: u32, x_end: u32, max_column_samples: u32) -> Vec { + let effective_width = x_end.saturating_sub(x_start).max(1); + let column_samples = effective_width.min(max_column_samples).max(1); + let mut columns = Vec::with_capacity(column_samples as usize); + + for column in 0..column_samples { + columns.push(evenly_spaced_sample(x_start, x_end, column, column_samples)); + } + + columns +} + fn overlap_global_informative_span(left: &RgbaImage, right: &RgbaImage) -> Option { let left_span = informative_column_span(left, 0, left.height()); let right_span = informative_column_span(right, 0, right.height()); let width = left.width().min(right.width()); - - match (left_span, right_span) { + let structural_span = match (left_span, right_span) { (Some(left_span), Some(right_span)) => { let start_x = left_span.start_x.max(right_span.start_x); let end_exclusive_x = left_span.end_exclusive_x.min(right_span.end_exclusive_x).min(width); - (end_exclusive_x > start_x).then_some(InformativeSpan { start_x, end_exclusive_x }) + (end_exclusive_x > start_x).then_some(InformativeSpan { start_x, end_exclusive_x })? }, (Some(span), None) | (None, Some(span)) => { let end_exclusive_x = span.end_exclusive_x.min(width).max(span.start_x + 1); (end_exclusive_x > span.start_x) - .then_some(InformativeSpan { start_x: span.start_x, end_exclusive_x }) + .then_some(InformativeSpan { start_x: span.start_x, end_exclusive_x })? }, - (None, None) => None, + (None, None) => return None, + }; + + motion_coverage_supports_structural_span(left, right, structural_span) + .then_some(structural_span) +} + +fn motion_coverage_supports_structural_span( + left: &RgbaImage, + right: &RgbaImage, + structural_span: InformativeSpan, +) -> bool { + let width = left.width().min(right.width()); + let height = left.height().min(right.height()); + let x_start = structural_span.start_x.min(width.saturating_sub(1)); + let x_end = structural_span.end_exclusive_x.min(width).max(x_start.saturating_add(1)); + + if width == 0 || height == 0 { + return false; + } + + let row_samples = height.min(INFORMATIVE_SPAN_ROW_SAMPLES.max(2)).max(2); + let mut scores = Vec::with_capacity(width as usize); + let mut max_structure_score = 0_u32; + let mut max_motion_score = 0_u32; + + for x in 0..width { + let mut structure_score = 0_u32; + let mut motion_score = 0_u32; + + for row in 0..row_samples { + let y = evenly_spaced_sample(0, height, row, row_samples); + let next_y = y.saturating_add(1).min(height.saturating_sub(1)); + let left_pixel = left.get_pixel(x, y).0; + let right_pixel = right.get_pixel(x, y).0; + let left_next_pixel = left.get_pixel(x, next_y).0; + let right_next_pixel = right.get_pixel(x, next_y).0; + + motion_score = motion_score + .saturating_add(u32::from(left_pixel[0].abs_diff(right_pixel[0]))) + .saturating_add(u32::from(left_pixel[1].abs_diff(right_pixel[1]))) + .saturating_add(u32::from(left_pixel[2].abs_diff(right_pixel[2]))); + structure_score = structure_score + .saturating_add(u32::from(left_pixel[0].abs_diff(left_next_pixel[0]))) + .saturating_add(u32::from(left_pixel[1].abs_diff(left_next_pixel[1]))) + .saturating_add(u32::from(left_pixel[2].abs_diff(left_next_pixel[2]))) + .saturating_add(u32::from(right_pixel[0].abs_diff(right_next_pixel[0]))) + .saturating_add(u32::from(right_pixel[1].abs_diff(right_next_pixel[1]))) + .saturating_add(u32::from(right_pixel[2].abs_diff(right_next_pixel[2]))); + } + + max_structure_score = max_structure_score.max(structure_score); + max_motion_score = max_motion_score.max(motion_score); + + scores.push((structure_score, motion_score)); + } + + if max_structure_score == 0 || max_motion_score == 0 { + return false; + } + if raw_frame_pair_has_static_informative_band(&scores, max_structure_score, max_motion_score) { + return false; + } + + let structure_threshold = (max_structure_score / 8).max(1); + let motion_threshold = (max_motion_score / 8).max(1); + let span_scores = &scores[x_start as usize..x_end as usize]; + let mut informative_columns = 0_u32; + let mut moving_informative_columns = 0_u32; + + if raw_frame_pair_has_static_informative_edge( + span_scores, + structure_threshold, + motion_threshold, + x_start, + width.saturating_sub(x_end), + ) { + return false; + } + + for &(structure_score, motion_score) in span_scores { + if structure_score < structure_threshold { + continue; + } + + informative_columns = informative_columns.saturating_add(1); + + if motion_score >= motion_threshold { + moving_informative_columns = moving_informative_columns.saturating_add(1); + } + } + + informative_columns >= MOTION_COVERAGE_MIN_INFORMATIVE_COLUMNS + && moving_informative_columns.saturating_mul(100) + >= informative_columns.saturating_mul(MOTION_COVERAGE_MIN_PERCENT) +} + +fn raw_frame_pair_has_static_informative_band( + scores: &[(u32, u32)], + max_structure_score: u32, + max_motion_score: u32, +) -> bool { + if scores.len() < MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS + || max_structure_score == 0 + || max_motion_score == 0 + { + return false; + } + + let structure_threshold = + (max_structure_score / MOTION_COVERAGE_STATIC_BAND_STRUCTURE_DIVISOR).max(1); + let motion_threshold = (max_motion_score / MOTION_COVERAGE_STATIC_BAND_MOTION_DIVISOR).max(1); + let moving_motion_threshold = motion_threshold.saturating_add(1); + let mut moving_start = None; + let mut moving_end = None; + let mut static_flags = Vec::with_capacity(scores.len()); + + for (column, (structure_score, motion_score)) in scores.iter().enumerate() { + if *structure_score >= structure_threshold && *motion_score >= moving_motion_threshold { + moving_start.get_or_insert(column); + + moving_end = Some(column.saturating_add(1)); + } + + static_flags + .push(*structure_score >= structure_threshold && *motion_score <= motion_threshold); + } + + let Some(moving_start) = moving_start else { + return false; + }; + let Some(moving_end) = moving_end else { + return false; + }; + let mut static_columns = static_flags[..MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS] + .iter() + .filter(|is_static| **is_static) + .count(); + + if static_side_band_has_enough_columns(static_columns, 0, moving_start, moving_end) { + return true; } + + for end in MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS..static_flags.len() { + if static_flags[end - MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS] { + static_columns = static_columns.saturating_sub(1); + } + if static_flags[end] { + static_columns = static_columns.saturating_add(1); + } + + let start = end.saturating_add(1).saturating_sub(MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS); + + if static_side_band_has_enough_columns(static_columns, start, moving_start, moving_end) { + return true; + } + } + + false +} + +fn static_side_band_has_enough_columns( + static_columns: usize, + start: usize, + moving_start: usize, + moving_end: usize, +) -> bool { + let end = start.saturating_add(MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS); + let enough_static_columns = (static_columns as u32).saturating_mul(100) + >= (MOTION_COVERAGE_STATIC_BAND_MIN_COLUMNS as u32) + .saturating_mul(MOTION_COVERAGE_STATIC_BAND_MIN_PERCENT); + let side_of_moving_span = end <= moving_start || start >= moving_end; + + enough_static_columns && side_of_moving_span +} + +fn raw_frame_pair_has_static_informative_edge( + scores: &[(u32, u32)], + structure_threshold: u32, + motion_threshold: u32, + left_leading_columns: u32, + right_leading_columns: u32, +) -> bool { + raw_static_edge_run_len( + scores.iter().copied(), + structure_threshold, + motion_threshold, + left_leading_columns, + ) >= MOTION_COVERAGE_STATIC_EDGE_MIN_COLUMNS + || raw_static_edge_run_len( + scores.iter().rev().copied(), + structure_threshold, + motion_threshold, + right_leading_columns, + ) >= MOTION_COVERAGE_STATIC_EDGE_MIN_COLUMNS +} + +fn raw_static_edge_run_len( + iter: I, + structure_threshold: u32, + motion_threshold: u32, + leading_columns: u32, +) -> u32 +where + I: IntoIterator, +{ + let mut skipped_columns = leading_columns; + let mut static_columns = 0_u32; + let mut seen_informative = false; + + for (structure_score, motion_score) in iter { + if structure_score < structure_threshold { + if seen_informative { + break; + } + + skipped_columns = skipped_columns.saturating_add(1); + + if skipped_columns > MOTION_COVERAGE_STATIC_EDGE_MAX_LEADING_COLUMNS { + return 0; + } + + continue; + } + if skipped_columns > MOTION_COVERAGE_STATIC_EDGE_MAX_LEADING_COLUMNS { + return 0; + } + + seen_informative = true; + + if motion_score >= motion_threshold { + break; + } + + static_columns = static_columns.saturating_add(1); + } + + static_columns } diff --git a/packages/rsnap-overlay/src/scroll_capture/tests.rs b/packages/rsnap-overlay/src/scroll_capture/tests.rs index 47953520..052f3962 100644 --- a/packages/rsnap-overlay/src/scroll_capture/tests.rs +++ b/packages/rsnap-overlay/src/scroll_capture/tests.rs @@ -39,6 +39,116 @@ fn make_browser_like_window(width: u32, height: u32, start_row: u32) -> image::R test_support::make_browser_like_window(width, height, start_row) } +fn make_unregistered_composited_frame(width: u32, height: u32, seed: u32) -> image::RgbaImage { + image::RgbaImage::from_fn(width, height, |x, y| { + let mut value = seed + .wrapping_add(x.wrapping_mul(0x045D_9F3B)) + .wrapping_add(y.wrapping_mul(0x9E37_79B9)); + + value ^= value >> 16; + value = value.wrapping_mul(0x85EB_CA6B); + value ^= value >> 13; + + Rgba([(value & 0xff) as u8, ((value >> 8) & 0xff) as u8, ((value >> 16) & 0xff) as u8, 255]) + }) +} + +fn make_static_sidebar_center_frame( + width: u32, + height: u32, + center_start_x: u32, + center_width: u32, + center_start_row: u32, + center_seed: u32, + center_scrolls: bool, +) -> image::RgbaImage { + image::RgbaImage::from_fn(width, height, |x, y| { + let in_center = x >= center_start_x && x < center_start_x.saturating_add(center_width); + + if !in_center { + let stripe = (y % 32) as u8; + + return Rgba([ + stripe.saturating_mul(5), + 80_u8.saturating_add(stripe), + 180_u8.saturating_sub(stripe.saturating_mul(2)), + 255, + ]); + } + + let document_row = if center_scrolls { center_start_row.saturating_add(y) } else { y }; + let mut value = center_seed + .wrapping_add(document_row.wrapping_mul(0x9E37_79B9)) + .wrapping_add(x.wrapping_mul(0x85EB_CA6B)); + + value ^= value >> 16; + value = value.wrapping_mul(0xC2B2_AE35); + value ^= value >> 13; + + Rgba([(value & 0xff) as u8, ((value >> 8) & 0xff) as u8, ((value >> 16) & 0xff) as u8, 255]) + }) +} + +fn make_codex_like_right_static_rail_frame( + width: u32, + height: u32, + center_start_row: u32, +) -> image::RgbaImage { + let left_blank_width = width / 10; + let right_rail_width = width / 5; + let right_rail_start = width.saturating_sub(right_rail_width); + + image::RgbaImage::from_fn(width, height, |x, y| { + if x < left_blank_width { + return Rgba([24, 24, 28, 255]); + } + if x >= right_rail_start { + let local_x = x.saturating_sub(right_rail_start); + let row = y / 34; + let y_in_row = y % 34; + let inside_block = local_x > 12 + && local_x < right_rail_width.saturating_sub(12) + && (5..=25).contains(&y_in_row); + let text_marker = inside_block && (8..=15).contains(&y_in_row) && local_x % 29 < 14; + let base = if inside_block { 34 + ((row % 5) as u8).saturating_mul(5) } else { 25 }; + let marker = if text_marker { 48 } else { 0 }; + let value = base.saturating_add(marker); + + return Rgba([value, value, value.saturating_add(4), 255]); + } + + let document_row = center_start_row.saturating_add(y); + let local_x = x.saturating_sub(left_blank_width); + let band = ((document_row / 22) % 11) as u8; + let mut value = document_row + .wrapping_mul(0x9E37_79B9) + .wrapping_add(local_x.wrapping_mul(0x85EB_CA6B)) + .wrapping_add((band as u32).wrapping_mul(0x045D_9F3B)); + + value ^= value >> 16; + value = value.wrapping_mul(0xC2B2_AE35); + value ^= value >> 13; + + Rgba([(value & 0xff) as u8, ((value >> 8) & 0xff) as u8, ((value >> 16) & 0xff) as u8, 255]) + }) +} + +fn make_dense_unique_scroll_frame(width: u32, height: u32, start_row: u32) -> image::RgbaImage { + image::RgbaImage::from_fn(width, height, |x, y| { + let document_row = start_row.saturating_add(y); + let mut value = document_row + .wrapping_mul(0x9E37_79B9) + .wrapping_add(x.wrapping_mul(0x85EB_CA6B)) + .wrapping_add(document_row.rotate_left(13) ^ x.rotate_left(7)); + + value ^= value >> 16; + value = value.wrapping_mul(0xC2B2_AE35); + value ^= value >> 13; + + Rgba([(value & 0xff) as u8, ((value >> 8) & 0xff) as u8, ((value >> 16) & 0xff) as u8, 255]) + }) +} + #[cfg(target_os = "macos")] fn build_worker_pairwise_session(frame: image::RgbaImage) -> ScrollSession { ScrollSession::new(frame, 320).expect("worker pairwise test session should initialize") @@ -278,6 +388,347 @@ fn session_tracks_large_slowdown_after_first_growth() { assert_eq!(session.current_viewport_top_y(), 314); } +#[test] +fn burst_motion_hint_does_not_override_underconsumed_visual_authority() { + let mut session = ScrollSession::new(make_sparse_textlike_window(256, 240, 0), 320).unwrap(); + + assert_eq!( + session + .observe_downward_growth_to_viewport( + make_sparse_textlike_window(256, 240, 16), + 16, + true, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows: 16 }), + "test_tiny_smooth_scroll_commit", + ) + .unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 16 } + ); + + for start_row in [96, 112, 128, 144] { + let _ = session + .observe_downward_sample_with_motion_hint_and_burst( + make_sparse_textlike_window(256, 240, start_row), + Some(224), + true, + ) + .unwrap(); + } + + let telemetry = session.commit_telemetry(); + + assert_eq!(telemetry.sample_eval_effective_motion_rows_hint, Some(224)); + assert_eq!(session.current_viewport_top_y(), 16); + assert_eq!(session.export_image().height(), 256); +} + +#[test] +fn initial_smooth_scroll_burst_can_commit_strong_match_beyond_underreported_hint() { + let document = (0_u32..1_024) + .map(|row| { + let mut value = row.wrapping_mul(0x9E37_79B9); + + value ^= value >> 16; + value = value.wrapping_mul(0x85EB_CA6B); + value ^= value >> 13; + + [(value & 0xff) as u8, ((value >> 8) & 0xff) as u8, ((value >> 16) & 0xff) as u8, 255] + }) + .collect::>(); + let mut session = ScrollSession::new(make_window(&document, 12, 0, 640), 320).unwrap(); + + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + make_window(&document, 12, 300, 640), + Some(42), + true, + ) + .unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 300 } + ); + assert_eq!(session.current_viewport_top_y(), 300); +} + +#[test] +fn transient_burst_input_hint_does_not_commit_without_visual_match() { + let mut session = ScrollSession::new(make_sparse_textlike_window(256, 240, 0), 320).unwrap(); + + assert_eq!( + session + .observe_downward_growth_to_viewport( + make_sparse_textlike_window(256, 240, 96), + 96, + true, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows: 96 }), + "test_initial_committed_growth", + ) + .unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 96 } + ); + + let composited_smooth_frame = make_unregistered_composited_frame(256, 240, 180); + + for _ in 0..2 { + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame.clone(), + Some(84), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + } + + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame, + Some(84), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + + let telemetry = session.commit_telemetry(); + + assert_eq!(telemetry.last_commit_decision_source, Some("test_initial_committed_growth")); + assert_eq!(telemetry.growth_commit_count, 1); + assert_eq!(session.current_viewport_top_y(), 96); + assert_eq!(session.export_image().height(), 336); +} + +#[test] +fn initial_transient_burst_input_hint_waits_for_visual_match() { + let composited_smooth_frame = make_unregistered_composited_frame(256, 240, 180); + let mut session = ScrollSession::new(make_sparse_textlike_window(256, 240, 0), 320).unwrap(); + + for _ in 0..2 { + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame.clone(), + Some(96), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + } + + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame, + Some(96), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + + let telemetry = session.commit_telemetry(); + + assert_eq!(telemetry.last_commit_decision_source, None); + assert_eq!(telemetry.growth_commit_count, 0); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image().height(), 240); +} + +#[test] +fn large_transient_burst_input_hint_waits_for_visual_match() { + let composited_smooth_frame = make_unregistered_composited_frame(256, 640, 520); + let mut session = ScrollSession::new(make_sparse_textlike_window(256, 640, 0), 320).unwrap(); + + for _ in 0..2 { + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame.clone(), + Some(432), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + } + + assert_eq!( + session + .observe_downward_sample_with_motion_hint_and_burst( + composited_smooth_frame, + Some(432), + true, + ) + .unwrap(), + ScrollObserveOutcome::PreviewUpdated + ); + + let telemetry = session.commit_telemetry(); + + assert_eq!(telemetry.last_commit_decision_source, None); + assert_eq!(telemetry.growth_commit_count, 0); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image().height(), 640); +} + +#[test] +fn static_sidebars_do_not_drive_downward_stitching_without_center_match() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); + let unrelated_center = make_static_sidebar_center_frame(320, 240, 145, 30, 96, 2, false); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + let outcome = session + .observe_downward_sample_with_motion_hint_and_burst(unrelated_center, Some(96), true) + .unwrap(); + + assert!(matches!( + outcome, + ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); + assert_eq!(session.commit_telemetry().growth_commit_count, 0); +} + +#[test] +fn dynamic_scroll_center_does_not_stitch_when_static_sidebars_are_in_selection() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); + let moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + let outcome = + session.observe_downward_sample_with_motion_hint_and_burst(moved, Some(72), true).unwrap(); + + assert!(matches!( + outcome, + ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); + assert_eq!(session.commit_telemetry().growth_commit_count, 0); +} + +#[test] +fn pairwise_shift_estimate_rejects_narrow_dynamic_center_with_static_sidebars_present() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); + let moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); + + assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &moved), None); +} + +#[test] +fn pairwise_shift_estimate_rejects_wide_dynamic_center_with_static_right_rail_present() { + let base = make_codex_like_right_static_rail_frame(640, 360, 0); + let moved = make_codex_like_right_static_rail_frame(640, 360, 72); + + assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &moved), None); +} + +#[test] +fn pairwise_shift_estimate_ignores_static_sidebars_without_center_scroll_match() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); + let unrelated_center = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 2, false); + + assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &unrelated_center), None); +} + +#[cfg(target_os = "macos")] +#[test] +fn worker_pairwise_vision_rejects_narrow_dynamic_center_with_static_sidebars_present() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); + let moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + assert_ne!( + session.observe_worker_pairwise_vision_frame(moved).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } + ); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); +} + +#[cfg(target_os = "macos")] +#[test] +fn worker_pairwise_vision_rejects_wide_dynamic_center_with_static_right_rail_present() { + let base = make_codex_like_right_static_rail_frame(640, 360, 0); + let moved = make_codex_like_right_static_rail_frame(640, 360, 72); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + assert_ne!( + session.observe_worker_pairwise_vision_frame(moved).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } + ); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); +} + +#[cfg(target_os = "macos")] +#[test] +fn worker_pairwise_vision_ignores_static_sidebars_without_center_scroll_match() { + let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); + let unrelated_center = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 2, false); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + assert_ne!( + session.observe_worker_pairwise_vision_frame(unrelated_center).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } + ); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); +} + +#[test] +fn transient_burst_visual_match_underconsuming_large_input_hint_does_not_commit() { + let document = (0..512_u32) + .map(|row| { + let value = row.wrapping_mul(37).wrapping_add(row.rotate_left(5)) as u8; + + [value, value.wrapping_mul(3), value.wrapping_add(91), 255] + }) + .collect::>(); + let mut session = ScrollSession::new(make_window(&document, 256, 0, 240), 320).unwrap(); + + assert_eq!( + session + .observe_downward_growth_to_viewport( + make_window(&document, 256, 96, 240), + 96, + true, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows: 96 }), + "test_initial_committed_growth", + ) + .unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 96 } + ); + + session.transient_burst_search_enabled = true; + session.transient_motion_rows_hint = Some(168); + + let candidate = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 97, + motion_rows: 1, + mean_abs_diff_x100: 0, + }; + + assert_eq!( + session + .block_invalid_downward_candidate( + &make_window(&document, 256, 97, 240), + 1, + candidate, + true + ) + .unwrap(), + Some(ScrollObserveOutcome::PreviewUpdated) + ); + assert_eq!(session.current_viewport_top_y(), 96); + assert_eq!(session.export_image().height(), 336); + assert_eq!(session.last_block_reason(), Some("visual_motion_underconsumed_input_hint")); +} + #[cfg(target_os = "macos")] #[test] fn worker_pairwise_vision_commits_substantial_downward_growth_with_corroboration() { @@ -332,7 +783,7 @@ fn pairwise_downward_shift_estimate_tracks_successive_browser_like_steps() { } #[test] -fn trusted_pairwise_downward_shift_blocks_periodic_ambiguity_near_vision_motion() { +fn pairwise_shift_estimate_fails_closed_when_periodic_content_does_not_visibly_change() { let document: Vec<[u8; 4]> = (0..256) .map(|row| { let bucket = (row % 24) as u8; @@ -348,7 +799,7 @@ fn trusted_pairwise_downward_shift_blocks_periodic_ambiguity_near_vision_motion( let base = make_window(&document, 8, 0, 96); let moved = make_window(&document, 8, 24, 96); - assert!(support::estimate_pairwise_downward_shift_rows(&base, &moved).is_some()); + assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &moved), None); assert_eq!( support::trusted_pairwise_downward_shift_rows_near_motion(&base, &moved, 24, 24), None @@ -543,6 +994,46 @@ fn worker_pairwise_vision_handles_repeated_browser_like_frame_between_growth_ste ); } +#[cfg(target_os = "macos")] +#[test] +fn worker_pairwise_vision_catches_up_from_committed_frontier_after_reacquire_block() { + let base = make_dense_unique_scroll_frame(512, 640, 0); + let first = make_dense_unique_scroll_frame(512, 640, 180); + let first_reference = first.clone(); + let catchup = make_dense_unique_scroll_frame(512, 640, 540); + let mut session = build_worker_pairwise_session(base.clone()); + let first_growth = assert_worker_pairwise_commit( + &mut session, + &base, + first, + "initial pairwise registration should detect downward motion", + ); + + assert_eq!( + support::trusted_pairwise_downward_shift_rows_near_motion( + &first_reference, + &catchup, + 360, + 24 + ), + Some(360) + ); + + session.worker_pairwise_requires_committed_reacquire = true; + + assert_eq!(session.current_viewport_top_y(), growth_rows_i32(first_growth)); + + let catchup_outcome = + session.observe_worker_pairwise_vision_frame_with_motion_hint(catchup, Some(360)).unwrap(); + + assert_eq!( + catchup_outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 360 } + ); + assert_eq!(session.current_viewport_top_y(), 540); + assert_eq!(session.export_image().height(), 640 + first_growth + 360); +} + #[cfg(target_os = "macos")] #[test] fn worker_pairwise_vision_browser_like_followup_does_not_commit_tail_after_blocked_overshot() { diff --git a/scripts/smoke/lib/pasteboard-image-info.swift b/scripts/smoke/lib/pasteboard-image-info.swift index a65b7db6..a8f1b9f6 100644 --- a/scripts/smoke/lib/pasteboard-image-info.swift +++ b/scripts/smoke/lib/pasteboard-image-info.swift @@ -8,6 +8,14 @@ while Date() <= deadline { if let image = NSImage(pasteboard: .general) { var proposedRect = CGRect(origin: .zero, size: image.size) if let cgImage = image.cgImage(forProposedRect: &proposedRect, context: nil, hints: nil) { + if let outputPath = ProcessInfo.processInfo.environment["PASTEBOARD_IMAGE_OUTPUT_PATH"], + outputPath.isEmpty == false + { + let bitmap = NSBitmapImageRep(cgImage: cgImage) + if let png = bitmap.representation(using: .png, properties: [:]) { + try? png.write(to: URL(fileURLWithPath: outputPath)) + } + } print("width=\(cgImage.width) height=\(cgImage.height)") exit(0) } diff --git a/scripts/smoke/lib/scroll-background-window.swift b/scripts/smoke/lib/scroll-background-window.swift index 69a50cf2..b8a219f4 100644 --- a/scripts/smoke/lib/scroll-background-window.swift +++ b/scripts/smoke/lib/scroll-background-window.swift @@ -1,9 +1,27 @@ import AppKit import Foundation +private enum ScrollBackgroundMode: String { + case fullDocument = "full_document" + case codexLike = "codex_like" +} + +private func readMode() -> ScrollBackgroundMode { + let raw = ProcessInfo.processInfo.environment["SCROLL_BACKGROUND_MODE"] ?? "" + + return ScrollBackgroundMode(rawValue: raw) ?? .fullDocument +} + +private func readProofStripeEnabled() -> Bool { + let raw = ProcessInfo.processInfo.environment["SCROLL_BACKGROUND_PROOF_STRIPE"] ?? "" + + return raw == "1" || raw.lowercased() == "true" +} + final class ScrollDocumentView: NSView { private let rowHeight: CGFloat = 72 private let rows = 80 + private let proofStripeEnabled = readProofStripeEnabled() private let textAttributes: [NSAttributedString.Key: Any] = [ .font: NSFont.monospacedSystemFont(ofSize: 19, weight: .medium), .foregroundColor: NSColor(calibratedWhite: 0.10, alpha: 1), @@ -63,6 +81,7 @@ final class ScrollDocumentView: NSView { withAttributes: smallTextAttributes ) } + drawProofStripe(in: dirtyRect) } private func markerColumns(width: CGFloat) -> [CGFloat] { @@ -80,6 +99,77 @@ final class ScrollDocumentView: NSView { max(220, width * 0.64 + 54), ].filter { $0 < width - 360 } } + + private func drawProofStripe(in dirtyRect: NSRect) { + guard proofStripeEnabled else { + return + } + let stripeBandHeight: CGFloat = 8 + let stripeWidth: CGFloat = 8 + let stripeX = max(0, min(bounds.width - stripeWidth, bounds.midX - stripeWidth / 2)) + let startY = max(0, Int(floor(dirtyRect.minY))) + let endY = min(Int(ceil(dirtyRect.maxY)), Int(ceil(bounds.height))) + + guard startY < endY else { + return + } + + for y in startY..> 8) & 0xff) / 255.0, + blue: 251.0 / 255.0, + alpha: 1 + ).setFill() + CGRect(x: stripeX, y: CGFloat(y), width: stripeWidth, height: 1).fill() + } + } +} + +final class StaticSidebarView: NSView { + private let side: String + private let titleAttributes: [NSAttributedString.Key: Any] = [ + .font: NSFont.monospacedSystemFont(ofSize: 17, weight: .semibold), + .foregroundColor: NSColor(calibratedWhite: 0.86, alpha: 1), + ] + private let rowAttributes: [NSAttributedString.Key: Any] = [ + .font: NSFont.monospacedSystemFont(ofSize: 13, weight: .medium), + .foregroundColor: NSColor(calibratedWhite: 0.62, alpha: 1), + ] + + init(frame frameRect: NSRect, side: String) { + self.side = side + + super.init(frame: frameRect) + wantsLayer = true + layer?.backgroundColor = NSColor(calibratedWhite: 0.11, alpha: 1).cgColor + } + + @available(*, unavailable) + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func draw(_ dirtyRect: NSRect) { + NSColor(calibratedWhite: 0.11, alpha: 1).setFill() + dirtyRect.fill() + + "\(side) static rail".draw(at: CGPoint(x: 24, y: 32), withAttributes: titleAttributes) + + for row in 0..<22 { + let y = 78 + CGFloat(row) * 34 + let rect = CGRect(x: 20, y: y, width: max(40, bounds.width - 40), height: 22) + let brightness = 0.16 + CGFloat(row % 4) * 0.018 + + NSColor(calibratedWhite: brightness, alpha: 1).setFill() + NSBezierPath(roundedRect: rect, xRadius: 5, yRadius: 5).fill() + "item \(String(format: "%02d", row))".draw( + at: CGPoint(x: rect.minX + 12, y: rect.minY + 3), + withAttributes: rowAttributes + ) + } + } } final class ScrollBackgroundDelegate: NSObject, NSApplicationDelegate { @@ -101,7 +191,39 @@ final class ScrollBackgroundDelegate: NSObject, NSApplicationDelegate { window.level = .floating window.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle] - let scrollView = NSScrollView(frame: CGRect(origin: .zero, size: frame.size)) + let mode = readMode() + let rootView = NSView(frame: CGRect(origin: .zero, size: frame.size)) + rootView.autoresizingMask = [.width, .height] + rootView.wantsLayer = true + rootView.layer?.backgroundColor = NSColor(calibratedWhite: 0.11, alpha: 1).cgColor + + let scrollFrame: CGRect + if mode == .codexLike { + let centerWidth = max(360, min(frame.width * 0.34, 720)) + let centerX = (frame.width - centerWidth) / 2 + let leftView = StaticSidebarView( + frame: CGRect(x: 0, y: 0, width: centerX, height: frame.height), + side: "left" + ) + let rightView = StaticSidebarView( + frame: CGRect( + x: centerX + centerWidth, + y: 0, + width: frame.width - centerX - centerWidth, + height: frame.height + ), + side: "right" + ) + leftView.autoresizingMask = [.height, .maxXMargin] + rightView.autoresizingMask = [.height, .minXMargin] + rootView.addSubview(leftView) + rootView.addSubview(rightView) + scrollFrame = CGRect(x: centerX, y: 0, width: centerWidth, height: frame.height) + } else { + scrollFrame = CGRect(origin: .zero, size: frame.size) + } + + let scrollView = NSScrollView(frame: scrollFrame) scrollView.autoresizingMask = [.width, .height] scrollView.hasVerticalScroller = true scrollView.hasHorizontalScroller = false @@ -113,10 +235,11 @@ final class ScrollBackgroundDelegate: NSObject, NSApplicationDelegate { let documentHeight = max(frame.height * 5, 5_760) let documentView = ScrollDocumentView( - frame: CGRect(x: 0, y: 0, width: frame.width, height: documentHeight) + frame: CGRect(x: 0, y: 0, width: scrollFrame.width, height: documentHeight) ) scrollView.documentView = documentView - window.contentView = scrollView + rootView.addSubview(scrollView) + window.contentView = rootView DistributedNotificationCenter.default().addObserver( self, diff --git a/scripts/smoke/lib/scroll-export-continuity.swift b/scripts/smoke/lib/scroll-export-continuity.swift new file mode 100644 index 00000000..0538a0f5 --- /dev/null +++ b/scripts/smoke/lib/scroll-export-continuity.swift @@ -0,0 +1,157 @@ +import AppKit +import Foundation + +func fail(_ message: String) -> Never { + fputs("[smoke] FAIL \(message)\n", stderr) + exit(1) +} + +guard CommandLine.arguments.count == 2 else { + fail("usage: scroll-export-continuity.swift ") +} + +let path = CommandLine.arguments[1] +guard + let image = NSImage(contentsOfFile: path), + let tiffData = image.tiffRepresentation, + let bitmap = NSBitmapImageRep(data: tiffData) +else { + fail("could not read export image at \(path)") +} + +let width = bitmap.pixelsWide +let height = bitmap.pixelsHigh +guard width > 0, height > 0 else { + fail("export image has invalid dimensions \(width)x\(height)") +} + +struct RGB { + let red: Double + let green: Double + let blue: Double +} + +func deviceRGB(_ color: NSColor) -> RGB { + guard let rgb = color.usingColorSpace(.deviceRGB) else { + return RGB(red: 0, green: 0, blue: 0) + } + + return RGB( + red: rgb.redComponent * 255.0, + green: rgb.greenComponent * 255.0, + blue: rgb.blueComponent * 255.0 + ) +} + +func pixelRGB(x: Int, y: Int) -> RGB { + guard let color = bitmap.colorAt(x: x, y: y)?.usingColorSpace(.deviceRGB) else { + return RGB(red: 0, green: 0, blue: 0) + } + + return RGB( + red: color.redComponent * 255.0, + green: color.greenComponent * 255.0, + blue: color.blueComponent * 255.0 + ) +} + +func distance(_ left: RGB, _ right: RGB) -> Double { + let red = left.red - right.red + let green = left.green - right.green + let blue = left.blue - right.blue + + return red * red + green * green + blue * blue +} + +let expectedRows: [RGB] = (0..<80).map { row in + let hue = CGFloat((row * 37) % 360) / 360.0 + return deviceRGB( + NSColor(calibratedHue: hue, saturation: 0.24, brightness: 0.97, alpha: 1) + ) +} +let sampleXs = Array( + Set([ + min(max(20, width / 120), width - 1), + max(0, width - min(max(20, width / 120), width)), + ]) +).sorted() + +struct Candidate { + let meanError: Double + let maxError: Double + let rowHeight: Int + let offset: Int + let startRow: Int + let bands: Int +} + +func score(rowHeight: Int, offset: Int, startRow: Int) -> Candidate? { + guard offset >= 0, offset < height else { + return nil + } + let bands = max(0, (height - offset) / rowHeight) + guard bands >= 4 else { + return nil + } + var total = 0.0 + var maxError = 0.0 + var samples = 0 + + for band in 0..= minimumBands else { + fail("row continuity coverage too low: bands=\(best.bands) height=\(height)") +} +guard best.meanError <= 1_500 && best.maxError <= 5_000 else { + fail( + "row continuity failed: meanError=\(Int(best.meanError)) maxError=\(Int(best.maxError)) rowHeight=\(best.rowHeight) offset=\(best.offset) startRow=\(best.startRow) bands=\(best.bands)" + ) +} + +let endRow = (best.startRow + best.bands - 1) % expectedRows.count +print( + "row_sequence_ok width=\(width) height=\(height) rowHeight=\(best.rowHeight) offset=\(best.offset) startRow=\(best.startRow) endRow=\(endRow) bands=\(best.bands) meanError=\(Int(best.meanError))" +) diff --git a/scripts/smoke/native-scroll-capture-macos.sh b/scripts/smoke/native-scroll-capture-macos.sh index 9dbadc40..48ff0a83 100755 --- a/scripts/smoke/native-scroll-capture-macos.sh +++ b/scripts/smoke/native-scroll-capture-macos.sh @@ -21,9 +21,14 @@ Useful overrides: SCROLL_START_METHOD=keyboard optional: keyboard,toolbar SCROLL_DELTA_Y=36 SCROLL_INTERVAL_MS=220 - MIN_SCROLL_COMMITS=3 - MIN_EXPORT_GROWTH_PX=180 + SCROLL_BACKGROUND_MODE=full_document|codex_like + SCROLL_START_SETTLE_S=5.0 + EXPECT_SCROLL_GROWTH=1 defaults to 0 for codex_like + MIN_SCROLL_COMMITS=2 defaults to 0 when EXPECT_SCROLL_GROWTH=0 + MIN_EXPORT_GROWTH_PX=180 defaults to 0 when EXPECT_SCROLL_GROWTH=0 + VALIDATE_SCROLL_EXPORT_CONTINUITY=0 set to 1 with SCROLL_BACKGROUND_PROOF_STRIPE=1 APP_POST_VERIFY_SETTLE_S=0 + POST_FREEZE_SETTLE_S=2.0 EOF } @@ -65,16 +70,46 @@ SCROLL_DRIVER="${SCROLL_DRIVER:-wheel}" SCROLL_START_METHOD="${SCROLL_START_METHOD:-keyboard}" SCROLL_DELTA_Y="${SCROLL_DELTA_Y:-36}" SCROLL_INTERVAL_MS="${SCROLL_INTERVAL_MS:-220}" -MIN_SCROLL_COMMITS="${MIN_SCROLL_COMMITS:-3}" -MIN_EXPORT_GROWTH_PX="${MIN_EXPORT_GROWTH_PX:-180}" +SCROLL_START_SETTLE_S="${SCROLL_START_SETTLE_S:-5.0}" +SCROLL_BACKGROUND_MODE="${SCROLL_BACKGROUND_MODE:-full_document}" +SCROLL_BACKGROUND_PROOF_STRIPE="${SCROLL_BACKGROUND_PROOF_STRIPE:-0}" +if [[ -z "${EXPECT_SCROLL_GROWTH:-}" ]]; then + if [[ "$SCROLL_BACKGROUND_MODE" == "codex_like" ]]; then + EXPECT_SCROLL_GROWTH=0 + else + EXPECT_SCROLL_GROWTH=1 + fi +fi +if [[ -z "${MIN_SCROLL_COMMITS:-}" ]]; then + if [[ "$EXPECT_SCROLL_GROWTH" == "0" ]]; then + MIN_SCROLL_COMMITS=0 + else + MIN_SCROLL_COMMITS=2 + fi +fi +if [[ -z "${MIN_EXPORT_GROWTH_PX:-}" ]]; then + if [[ "$EXPECT_SCROLL_GROWTH" == "0" ]]; then + MIN_EXPORT_GROWTH_PX=0 + else + MIN_EXPORT_GROWTH_PX=180 + fi +fi APP_POST_VERIFY_SETTLE_S="${APP_POST_VERIFY_SETTLE_S:-0}" +VALIDATE_SCROLL_EXPORT_CONTINUITY="${VALIDATE_SCROLL_EXPORT_CONTINUITY:-0}" OVERLAY_SETTLE_S="${OVERLAY_SETTLE_S:-0.10}" -POST_FREEZE_SETTLE_S="${POST_FREEZE_SETTLE_S:-0.16}" +POST_FREEZE_SETTLE_S="${POST_FREEZE_SETTLE_S:-2.0}" POST_COPY_SETTLE_S="${POST_COPY_SETTLE_S:-0.30}" PATH_CYCLES="${PATH_CYCLES:-1}" +EXPORT_IMAGE_PATH_WAS_PROVIDED=0 +if [[ -n "${EXPORT_IMAGE_PATH:-}" ]]; then + EXPORT_IMAGE_PATH_WAS_PROVIDED=1 +else + EXPORT_IMAGE_PATH="$(mktemp "${TMPDIR:-/tmp}/rsnap-scroll-export.XXXXXX.png")" +fi if [[ -z "${POST_SCROLL_SETTLE_S:-}" ]]; then POST_SCROLL_SETTLE_S=2.20 fi +export SCROLL_BACKGROUND_MODE SCROLL_BACKGROUND_PROOF_STRIPE restore_preferences() { live_hud_stop_awake_assertion @@ -90,6 +125,9 @@ restore_preferences() { done fi rm -f "$PREF_SNAPSHOT" "$SCROLL_BACKGROUND_READY" + if [[ "$EXPORT_IMAGE_PATH_WAS_PROVIDED" != "1" ]]; then + rm -f "$EXPORT_IMAGE_PATH" + fi } if defaults export "$PREF_DOMAIN" "$PREF_SNAPSHOT" >/dev/null 2>&1; then @@ -236,13 +274,14 @@ start_scroll_background() { parse_telemetry() { local log_path="$1" - python3 - "$log_path" "$MIN_SCROLL_COMMITS" "$MIN_EXPORT_GROWTH_PX" "$SCROLL_START_METHOD" <<'PY' + python3 - "$log_path" "$MIN_SCROLL_COMMITS" "$MIN_EXPORT_GROWTH_PX" "$SCROLL_START_METHOD" "$EXPECT_SCROLL_GROWTH" <<'PY' import re import sys -path, min_commits_raw, min_growth_raw, start_method = sys.argv[1:5] +path, min_commits_raw, min_growth_raw, start_method, expect_growth_raw = sys.argv[1:6] min_commits = int(min_commits_raw) min_growth = int(min_growth_raw) +expect_growth = expect_growth_raw != "0" expected_start_source = { "keyboard": "keyboard_s", "toolbar": "toolbar", @@ -264,6 +303,7 @@ manual_mode = bool( text, ) ) +input_ready = "event=capture.scroll_input_ready" in text tap_not_used = bool( re.search( r"event=capture\.scroll_input_tap\b[^\n]*outcome=not_used\b", @@ -304,7 +344,8 @@ growth = max_height - base_height print( f"[smoke] telemetry froze={froze} handoff={handoff} " f"entry_started={entry_started} start_source={expected_start_source} " - f"started={started} manual_mode={manual_mode} sampled={sampled} commits={len(commits)} " + f"started={started} manual_mode={manual_mode} input_ready={input_ready} " + f"sampled={sampled} commits={len(commits)} " f"tap_not_used={tap_not_used} wheel_intercepted={wheel_intercepted} " f"wheel_observed={wheel_observed} " f"max_export_height={max_height} base_height={base_height} growth={growth} " @@ -322,18 +363,26 @@ if not started: failures.append("scroll capture did not start") if not manual_mode: failures.append("scroll capture did not use universal manual mode") +if not input_ready: + failures.append("scroll capture input did not become ready") if not tap_not_used: - failures.append("scroll capture did not use overlay-local wheel forwarding") + failures.append("scroll capture did not use passthrough/global wheel monitoring") if not wheel_input_seen: failures.append("scroll capture did not receive wheel input") if not sampled: failures.append("scroll capture did not sample") if base_height <= 0: failures.append("scroll capture start height was not recorded") -if len(commits) < min_commits: - failures.append(f"committed growth count {len(commits)} < {min_commits}") -if growth < min_growth: - failures.append(f"export growth {growth}px < {min_growth}px") +if expect_growth: + if len(commits) < min_commits: + failures.append(f"committed growth count {len(commits)} < {min_commits}") + if growth < min_growth: + failures.append(f"export growth {growth}px < {min_growth}px") +else: + if len(commits) != 0: + failures.append(f"expected fail-closed no growth, but committed growth count was {len(commits)}") + if growth != 0: + failures.append(f"expected fail-closed no export growth, but growth was {growth}px") if auto_event: failures.append("unexpected legacy auto-scroll telemetry") @@ -390,6 +439,15 @@ PY smoke_log "display bounds: $DISPLAY_BOUNDS" smoke_log "drag points: $DRAG_POINTS scroll_point=$SCROLL_POINT base_height=$BASE_HEIGHT" +smoke_log "background_mode=$SCROLL_BACKGROUND_MODE expect_growth=$EXPECT_SCROLL_GROWTH min_commits=$MIN_SCROLL_COMMITS min_growth=$MIN_EXPORT_GROWTH_PX" +case "$SCROLL_BACKGROUND_MODE" in + full_document|codex_like) + ;; + *) + echo "[smoke] FAIL unknown SCROLL_BACKGROUND_MODE=$SCROLL_BACKGROUND_MODE" >&2 + exit 2 + ;; +esac case "$SCROLL_START_METHOD" in keyboard|toolbar) ;; @@ -427,7 +485,7 @@ case "$SCROLL_START_METHOD" in click_scroll_toolbar_icon ;; esac -sleep 0.25 +sleep "$SCROLL_START_SETTLE_S" case "$SCROLL_DRIVER" in notification) @@ -448,10 +506,10 @@ case "$SCROLL_DRIVER" in exit 2 ;; esac +sleep "$POST_SCROLL_SETTLE_S" if [[ "$SCROLL_DRIVER" == "notification" || "$SCROLL_DRIVER" == "wheel" ]]; then assert_scroll_background_moved fi -sleep "$POST_SCROLL_SETTLE_S" live_hud_focus_rsnap_overlay osascript -e 'set the clipboard to ""' >/dev/null @@ -461,6 +519,7 @@ pasteboard_ok=0 pasteboard_info="" if pasteboard_info="$( PASTEBOARD_WAIT_MS="${PASTEBOARD_WAIT_MS:-5000}" \ + PASTEBOARD_IMAGE_OUTPUT_PATH="$EXPORT_IMAGE_PATH" \ swift "$SCRIPT_DIR/lib/pasteboard-image-info.swift" 2>&1 )"; then pasteboard_ok=1 @@ -477,4 +536,12 @@ if [[ "$pasteboard_ok" != "1" ]]; then echo "[smoke] FAIL no scroll capture image was copied to the pasteboard" >&2 exit 1 fi +if [[ "$VALIDATE_SCROLL_EXPORT_CONTINUITY" == "1" && "$EXPECT_SCROLL_GROWTH" != "0" ]]; then + if [[ ! -s "$EXPORT_IMAGE_PATH" ]]; then + echo "[smoke] FAIL scroll capture export image was not saved for continuity validation" >&2 + exit 1 + fi + continuity_info="$(swift "$SCRIPT_DIR/lib/scroll-export-continuity.swift" "$EXPORT_IMAGE_PATH")" + smoke_log "export $continuity_info" +fi smoke_log "native scroll-capture smoke passed" From 271ccc64c049f2b237d5b37bb3129499f5be9642 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 11 May 2026 23:09:49 +0800 Subject: [PATCH 3/3] Fix scroll test helper cfg --- packages/rsnap-overlay/src/scroll_capture/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rsnap-overlay/src/scroll_capture/tests.rs b/packages/rsnap-overlay/src/scroll_capture/tests.rs index 052f3962..980152d5 100644 --- a/packages/rsnap-overlay/src/scroll_capture/tests.rs +++ b/packages/rsnap-overlay/src/scroll_capture/tests.rs @@ -133,6 +133,7 @@ fn make_codex_like_right_static_rail_frame( }) } +#[cfg(target_os = "macos")] fn make_dense_unique_scroll_frame(width: u32, height: u32, start_row: u32) -> image::RgbaImage { image::RgbaImage::from_fn(width, height, |x, y| { let document_row = start_row.saturating_add(y);