From b12cd7e3c6a3925e79246278192e595db483d531 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 14:34:24 +0800 Subject: [PATCH 01/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-rebuild","summary":"checkpoint worker pairwise scroll capture rebuild","intent":"preserve a rollback anchor after moving macOS downward stitching onto discrete screenshot plus worker-pairwise registration and fixing the trace-enabled screenshot decode crash while rare tearing remains under investigation","impact":"this lane now carries the screenshot-manager plus worker-pairwise scroll-capture rebuild, preview-export WYSIWYG cleanup, fresh trace analysis entrypoints, and the crash fix used by the latest touchpad live runs, but rare tearing still remains for follow-up","breaking":false,"risk":"medium","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"}]} --- Cargo.lock | 34 + Makefile.toml | 49 +- README.md | 44 +- apps/rsnap/src/app.rs | 32 +- apps/rsnap/src/app/capture.rs | 48 +- apps/rsnap/src/app/runtime.rs | 29 +- .../src/app/scroll_input_macos/decode.rs | 12 + .../rsnap/src/app/scroll_input_macos/state.rs | 125 +- apps/rsnap/src/app/scroll_input_macos/tap.rs | 20 + docs/guide/performance-checks.md | 60 +- docs/guide/scroll-capture-benchmarks.md | 25 + docs/spec/v0.md | 30 +- packages/rsnap-overlay/Cargo.toml | 5 + .../examples/scroll_capture_replay.rs | 181 + packages/rsnap-overlay/src/backend.rs | 191 +- packages/rsnap-overlay/src/lib.rs | 9 + .../src/live_frame_stream_macos.rs | 1721 +++- packages/rsnap-overlay/src/overlay.rs | 2629 +++++- .../src/overlay/replay_support.rs | 1517 ++++ .../src/overlay/scroll_runtime.rs | 751 +- .../src/overlay/session_state.rs | 57 +- .../src/overlay/trace_recording.rs | 739 ++ packages/rsnap-overlay/src/scroll_capture.rs | 7320 ++++++++++++++--- packages/rsnap-overlay/src/worker.rs | 33 +- scripts/scroll-capture-smoke-macos.sh | 545 -- 25 files changed, 13716 insertions(+), 2490 deletions(-) create mode 100644 packages/rsnap-overlay/examples/scroll_capture_replay.rs create mode 100644 packages/rsnap-overlay/src/overlay/replay_support.rs create mode 100644 packages/rsnap-overlay/src/overlay/trace_recording.rs delete mode 100755 scripts/scroll-capture-smoke-macos.sh diff --git a/Cargo.lock b/Cargo.lock index 9606dd30..7be5fdf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2872,6 +2872,16 @@ dependencies = [ "objc2-core-video", ] +[[package]] +name = "objc2-core-ml" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201b055e6acfa0f9f15568255d3f03ce2b54bc86d1814442dc69138e36813e18" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -3108,6 +3118,25 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-vision" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfc194758a2d5d7540b1ad283bfb9ca318ec608991892326e95b428230b2689b" +dependencies = [ + "block2 0.6.2", + "objc2 0.6.4", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-media", + "objc2-core-ml", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-image-io", +] + [[package]] name = "object" version = "0.37.3" @@ -3753,6 +3782,8 @@ dependencies = [ "color-eyre", "criterion", "device_query", + "directories", + "dispatch2", "egui", "egui-phosphor", "egui-wgpu", @@ -3767,11 +3798,14 @@ dependencies = [ "objc2-core-video", "objc2-foundation 0.3.2", "objc2-screen-capture-kit", + "objc2-vision", "pollster", "raw-window-handle", "serde", + "serde_json", "thiserror 2.0.18", "tracing", + "tracing-subscriber", "wgpu", "winit", "xcap", diff --git a/Makefile.toml b/Makefile.toml index 3f22f034..a6768410 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -130,37 +130,68 @@ args = [ # | ---------------------------- | --------- | --- | # | smoke-macos | composite | | # | smoke-self-check-macos | composite | | -# | smoke-scroll-capture-macos | command | | +# | replay-scroll-capture | command | | +# | replay-scroll-capture-self-check | command | | # | smoke-live-loupe-perf-macos | command | | -# | smoke-scroll-capture-self-check-macos | command | | # | smoke-live-loupe-self-check-macos | command | | +# | analyze-scroll-capture-trace | command | | [tasks.smoke-macos] workspace = false dependencies = [ "smoke-live-loupe-perf-macos", - "smoke-scroll-capture-macos", + "replay-scroll-capture", ] [tasks.smoke-self-check-macos] workspace = false dependencies = [ "smoke-live-loupe-self-check-macos", - "smoke-scroll-capture-self-check-macos", + "replay-scroll-capture-self-check", ] -[tasks.smoke-scroll-capture-macos] +[tasks.replay-scroll-capture] workspace = false -command = "scripts/scroll-capture-smoke-macos.sh" +command = "cargo" +args = [ + "run", + "-p", + "rsnap-overlay", + "--example", + "scroll_capture_replay", + "--", + "--force-worker-pairwise", +] + +[tasks.analyze-scroll-capture-trace] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "rsnap-overlay", + "--example", + "scroll_capture_replay", + "--", + "--force-worker-pairwise", + "--json", + "--summary-only", +] [tasks.smoke-live-loupe-perf-macos] workspace = false command = "scripts/live-loupe-perf-smoke-macos.sh" -[tasks.smoke-scroll-capture-self-check-macos] +[tasks.replay-scroll-capture-self-check] workspace = false -command = "scripts/scroll-capture-smoke-macos.sh" -args = ["--self-check"] +command = "cargo" +args = [ + "test", + "-p", + "rsnap-overlay", + "replay_recorded_live_trace_round_trips_one_commit", + "--lib", +] [tasks.smoke-live-loupe-self-check-macos] workspace = false diff --git a/README.md b/README.md index 13cd6e82..c46fe53c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ Prototype / in active development. ## Capture platform support -- Live sampling path: **macOS 12.3+** via ScreenCaptureKit (`SCStream`) stream samples. +- Live sampling path: **macOS 12.3+** via ScreenCaptureKit. Live loupe/window + sampling uses `SCStream`; downward scroll capture uses discrete + `SCScreenshotManager` region screenshots plus pairwise registration. - Live mode is stream-first and does not capture full display on cursor movement. - Frozen capture and scroll-capture imagery on macOS use the native capture stack; `docs/spec/v0.md` is the current contract source of truth. - Menubar and Dock are not included in live window-outline targeting. @@ -93,6 +95,7 @@ cargo run -p rsnap - In Frozen mode, use Cmd+S (macOS) / Ctrl+S to save a PNG to disk and exit. - After entering scroll capture from a dragged region on macOS, downward scrolling may append newly proven rows into the side preview. Upward scrolling never appends. Returning to already-stitched content should not grow the export; only newly proven content may be added. + The scroll-capture commit path uses discrete region screenshots plus pairwise image registration; clipboard and save must match the committed preview the user sees. `Space` copies the stitched image, Cmd+S (macOS) / Ctrl+S saves it, and `Esc` / `Back` returns to the original Frozen capture without exiting. - Output is configured in `settings.toml`: @@ -108,16 +111,47 @@ cargo make lint cargo make test ``` -macOS GUI smoke harnesses are also available: +Scroll-capture verification now starts with deterministic replay instead of the old GUI smoke: + +```sh +cargo make replay-scroll-capture +cargo make replay-scroll-capture-self-check +``` + +For semantic trace analysis (first bad frame, under-consumption, overshoot), use: + +```sh +cargo make analyze-scroll-capture-trace +``` + +The remaining macOS GUI smoke harnesses are still available for live-loupe and +desktop-session checks: ```sh cargo make smoke-self-check-macos cargo make smoke-macos ``` -These scripts drive a logged-in macOS desktop session, require the expected -Screen Recording / automation permissions, and are intended for dedicated smoke -verification runs rather than background CI on a shared desktop session. +`cargo make replay-scroll-capture` and `cargo make analyze-scroll-capture-trace` +now force the latest recorded live trace through the same worker-pairwise +commit path that current macOS production scroll capture uses. They are +trace-driven rather than scenario-driven, so they expect at least one recorded +trace under `~/Library/Application Support/ink.hack.rsnap/scroll-capture-traces/` +unless you pass `--trace ` directly to the example. Use the +direct example without `--force-worker-pairwise` only when you intentionally +want to compare the legacy recorded-source replay mode. `cargo make +replay-scroll-capture-self-check` is the repo-local fallback when you want to +verify the replay harness itself without relying on a user-recorded trace. +`cargo make smoke-self-check-macos` and `cargo make smoke-macos` still drive +the logged-in macOS live-loupe smoke path and require the expected Screen +Recording / automation permissions. + +For `XY-185` style downward scroll-capture work, treat the verification order as: + +1. deterministic tests and `cargo make check` +2. `cargo make replay-scroll-capture` +3. `cargo make analyze-scroll-capture-trace` +4. one fresh release live touchpad run with a newly recorded trace Repo-native performance entrypoints are available for deterministic benches and dedicated smoke: diff --git a/apps/rsnap/src/app.rs b/apps/rsnap/src/app.rs index 5371afff..9e56b926 100644 --- a/apps/rsnap/src/app.rs +++ b/apps/rsnap/src/app.rs @@ -6,10 +6,7 @@ mod scroll_input_macos; mod shell; #[cfg(target_os = "macos")] -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, -}; +use std::sync::Arc; use color_eyre::eyre::Result; use global_hotkey::{GlobalHotKeyEvent, GlobalHotKeyManager, hotkey::HotKey}; @@ -40,6 +37,8 @@ pub(crate) enum UserEvent { #[cfg(target_os = "macos")] OverlayStreamFrame, #[cfg(target_os = "macos")] + OverlayScrollInput, + #[cfg(target_os = "macos")] OverlayWorkerResponse, } @@ -71,8 +70,6 @@ struct App { #[cfg(target_os = "macos")] overlay_proxy: EventLoopProxy, #[cfg(target_os = "macos")] - overlay_stream_event_pending: Arc, - #[cfg(target_os = "macos")] scroll_input_observer_lifecycle: Arc, #[cfg(target_os = "macos")] scroll_input_shared_state: Arc, @@ -87,7 +84,6 @@ impl App { settings_hotkey: Option, hotkey_manager: Option, #[cfg(target_os = "macos")] overlay_proxy: EventLoopProxy, - #[cfg(target_os = "macos")] overlay_stream_event_pending: Arc, #[cfg(target_os = "macos")] scroll_input_observer_lifecycle: Arc< ScrollInputObserverLifecycle, >, @@ -121,8 +117,6 @@ impl App { #[cfg(target_os = "macos")] overlay_proxy, #[cfg(target_os = "macos")] - overlay_stream_event_pending, - #[cfg(target_os = "macos")] scroll_input_observer_lifecycle, #[cfg(target_os = "macos")] scroll_input_shared_state, @@ -187,23 +181,3 @@ impl App { pub fn run() -> Result<()> { runtime::run() } - -#[cfg(target_os = "macos")] -fn begin_coalesced_overlay_user_event_send(pending: &AtomicBool) -> bool { - pending.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_ok() -} - -#[cfg(test)] -mod tests { - #[cfg(target_os = "macos")] - use std::sync::atomic::AtomicBool; - - #[cfg(target_os = "macos")] - #[test] - fn begin_coalesced_overlay_user_event_send_only_allows_first_sender_per_flag() { - let pending = AtomicBool::new(false); - - assert!(super::begin_coalesced_overlay_user_event_send(&pending)); - assert!(!super::begin_coalesced_overlay_user_event_send(&pending)); - } -} diff --git a/apps/rsnap/src/app/capture.rs b/apps/rsnap/src/app/capture.rs index 4b373b17..4833fc5d 100644 --- a/apps/rsnap/src/app/capture.rs +++ b/apps/rsnap/src/app/capture.rs @@ -1,5 +1,5 @@ #[cfg(target_os = "macos")] -use std::sync::{Arc, atomic::Ordering}; +use std::sync::Arc; #[cfg(target_os = "macos")] use std::time::Duration; @@ -11,12 +11,12 @@ use winit::event_loop::ActiveEventLoop; use crate::app::App; #[cfg(target_os = "macos")] +use crate::app::UserEvent; +#[cfg(target_os = "macos")] use crate::app::scroll_input_macos::{ self, ScrollInputObserverLifecycle, ScrollInputObserverWaitOutcome, SharedScrollInputState, }; #[cfg(target_os = "macos")] -use crate::app::{self, UserEvent}; -#[cfg(target_os = "macos")] use crate::permissions_macos; use rsnap_overlay::{HudAnchor, OverlayConfig, OverlayControl, OverlayExit, OverlaySession}; @@ -107,19 +107,21 @@ impl App { #[cfg(target_os = "macos")] self.scroll_input_shared_state.clear(); + #[cfg(target_os = "macos")] + self.scroll_input_shared_state.set_event_waker(Some(Arc::new({ + let overlay_proxy = self.overlay_proxy.clone(); + + move || { + let _ = overlay_proxy.send_event(UserEvent::OverlayScrollInput); + } + }))); #[cfg(target_os = "macos")] overlay_session.set_scroll_frame_waker(Arc::new({ let overlay_proxy = self.overlay_proxy.clone(); - let overlay_stream_event_pending = Arc::clone(&self.overlay_stream_event_pending); move || { - if !app::begin_coalesced_overlay_user_event_send(&overlay_stream_event_pending) { - return; - } - if overlay_proxy.send_event(UserEvent::OverlayStreamFrame).is_err() { - overlay_stream_event_pending.store(false, Ordering::Release); - } + let _ = overlay_proxy.send_event(UserEvent::OverlayStreamFrame); } })); #[cfg(target_os = "macos")] @@ -170,6 +172,7 @@ impl App { #[cfg(target_os = "macos")] { self.scroll_input_shared_state.set_enabled(false); + self.scroll_input_shared_state.set_event_waker(None); self.scroll_input_shared_state.clear(); } @@ -226,6 +229,7 @@ impl App { #[cfg(target_os = "macos")] { self.scroll_input_shared_state.set_enabled(false); + self.scroll_input_shared_state.set_event_waker(None); self.scroll_input_shared_state.clear(); } @@ -275,6 +279,13 @@ impl App { shared_state: &Arc, observer_lifecycle: &Arc, ) -> Result<()> { + tracing::info!( + op = "scroll_input.prepare_start", + observer_status = ?observer_lifecycle.status(), + enabled = shared_state.is_enabled(), + "Preparing native scroll input for scroll capture." + ); + if observer_lifecycle.begin_start_if_needed() && let Err(err) = scroll_input_macos::spawn_scroll_input_observer( Arc::clone(shared_state), @@ -288,7 +299,16 @@ impl App { } match observer_lifecycle.wait_until_ready(SCROLL_INPUT_OBSERVER_READY_TIMEOUT) { - ScrollInputObserverWaitOutcome::Ready => Ok(()), + ScrollInputObserverWaitOutcome::Ready => { + tracing::info!( + op = "scroll_input.prepare_ready", + observer_status = ?observer_lifecycle.status(), + enabled = shared_state.is_enabled(), + "Native scroll input is ready for scroll capture." + ); + + Ok(()) + }, ScrollInputObserverWaitOutcome::TimedOut => Err(eyre::eyre!( "Scroll capture is still starting the native scroll observer. Retry once." )), @@ -302,6 +322,12 @@ impl App { fn enable_external_scroll_input(shared_state: &Arc) { shared_state.clear(); shared_state.set_enabled(true); + + tracing::info!( + op = "scroll_input.enabled", + enabled = shared_state.is_enabled(), + "Enabled native scroll input replay for scroll capture." + ); } pub(super) fn handle_overlay_control(&mut self, control: OverlayControl) { diff --git a/apps/rsnap/src/app/runtime.rs b/apps/rsnap/src/app/runtime.rs index 2ee9119e..041e8d2d 100644 --- a/apps/rsnap/src/app/runtime.rs +++ b/apps/rsnap/src/app/runtime.rs @@ -1,9 +1,6 @@ use std::collections::VecDeque; #[cfg(target_os = "macos")] -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, -}; +use std::sync::Arc; use std::time::{Duration, Instant}; use color_eyre::eyre; @@ -44,8 +41,6 @@ impl ApplicationHandler for App { UserEvent::TrayIcon => {}, #[cfg(target_os = "macos")] UserEvent::OverlayStreamFrame => { - self.overlay_stream_event_pending.store(false, Ordering::Release); - if let Some(session) = self.overlay_session.as_mut() { let control = session.handle_scroll_stream_frame_ready(); @@ -53,6 +48,14 @@ impl ApplicationHandler for App { } }, #[cfg(target_os = "macos")] + UserEvent::OverlayScrollInput => { + if let Some(session) = self.overlay_session.as_mut() { + let control = session.handle_scroll_input_ready(); + + self.handle_overlay_control(control); + } + }, + #[cfg(target_os = "macos")] UserEvent::OverlayWorkerResponse => { if let Some(session) = self.overlay_session.as_mut() { let control = session.handle_worker_response_ready(); @@ -177,9 +180,13 @@ impl ApplicationHandler for App { fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { if self.overlay_session.is_some() { - event_loop.set_control_flow(ControlFlow::WaitUntil( - Instant::now() + Duration::from_millis(16), - )); + let wait_interval = self + .overlay_session + .as_ref() + .map(|session| session.interactive_wait_interval()) + .unwrap_or_else(|| Duration::from_millis(16)); + + event_loop.set_control_flow(ControlFlow::WaitUntil(Instant::now() + wait_interval)); } else if self.settings_window.is_some() { event_loop.set_control_flow(ControlFlow::WaitUntil( Instant::now() + Duration::from_millis(250), @@ -258,8 +265,6 @@ pub(super) fn run() -> Result<()> { #[cfg(target_os = "macos")] let overlay_proxy: EventLoopProxy = event_loop.create_proxy(); #[cfg(target_os = "macos")] - let overlay_stream_event_pending = Arc::new(AtomicBool::new(false)); - #[cfg(target_os = "macos")] let scroll_input_observer_lifecycle = Arc::new(ScrollInputObserverLifecycle::default()); #[cfg(target_os = "macos")] let scroll_input_shared_state = Arc::new(SharedScrollInputState::default()); @@ -271,8 +276,6 @@ pub(super) fn run() -> Result<()> { #[cfg(target_os = "macos")] overlay_proxy, #[cfg(target_os = "macos")] - overlay_stream_event_pending, - #[cfg(target_os = "macos")] scroll_input_observer_lifecycle, #[cfg(target_os = "macos")] scroll_input_shared_state, diff --git a/apps/rsnap/src/app/scroll_input_macos/decode.rs b/apps/rsnap/src/app/scroll_input_macos/decode.rs index 8eb33ffa..7d393952 100644 --- a/apps/rsnap/src/app/scroll_input_macos/decode.rs +++ b/apps/rsnap/src/app/scroll_input_macos/decode.rs @@ -43,6 +43,18 @@ pub(super) fn decode_scroll_input_from_cg_event( let gesture_ended = scroll_phase_bits_are_terminal(scroll_phase) || scroll_phase_bits_are_terminal(momentum_phase); + tracing::info!( + op = "scroll_input.tap_decoded", + raw_delta_y, + global_x = location.x, + global_y = location.y, + scroll_phase, + momentum_phase, + gesture_active, + gesture_ended, + "Decoded native macOS scroll input event." + ); + decode_scroll_input_from_fields(raw_delta_y, location, gesture_active, gesture_ended) } diff --git a/apps/rsnap/src/app/scroll_input_macos/state.rs b/apps/rsnap/src/app/scroll_input_macos/state.rs index 29bc20f5..8830a068 100644 --- a/apps/rsnap/src/app/scroll_input_macos/state.rs +++ b/apps/rsnap/src/app/scroll_input_macos/state.rs @@ -1,21 +1,28 @@ use std::collections::VecDeque; use std::sync::{ - Condvar, Mutex, + Arc, Condvar, Mutex, atomic::{AtomicBool, AtomicU64, Ordering}, }; use std::time::{Duration, Instant}; -const SHARED_SCROLL_INPUT_QUEUE_CAPACITY: usize = 64; +const SHARED_SCROLL_INPUT_QUEUE_CAPACITY: usize = 512; #[derive(Default)] pub(in crate::app) struct SharedScrollInputState { enabled: AtomicBool, queue_state: Mutex, + event_waker: Mutex>, next_seq: AtomicU64, } impl SharedScrollInputState { pub(in crate::app) fn set_enabled(&self, enabled: bool) { self.enabled.store(enabled, Ordering::Release); + + tracing::info!( + op = "scroll_input.enabled_state_changed", + enabled, + "Updated native scroll input enabled state." + ); } pub(in crate::app) fn is_enabled(&self) -> bool { @@ -27,8 +34,24 @@ impl SharedScrollInputState { Ok(queue_state) => queue_state, Err(poisoned) => poisoned.into_inner(), }; + let cleared_events = queue_state.queue.len(); *queue_state = SharedScrollInputQueueState::default(); + + tracing::info!( + op = "scroll_input.queue_cleared", + cleared_events, + "Cleared queued native scroll input events." + ); + } + + pub(in crate::app) fn set_event_waker(&self, event_waker: Option) { + let mut waker_slot = match self.event_waker.lock() { + Ok(waker_slot) => waker_slot, + Err(poisoned) => poisoned.into_inner(), + }; + + *waker_slot = event_waker; } pub(in crate::app) fn record( @@ -92,6 +115,26 @@ impl SharedScrollInputState { queue_state.last_recorded = Some(event); + tracing::info!( + op = "scroll_input.queued", + seq, + delta_y = event.delta_y, + global_x = event.global_x, + global_y = event.global_y, + gesture_active = event.gesture_active, + gesture_ended = event.gesture_ended, + queue_len = queue_state.queue.len(), + "Queued native scroll input event for later overlay replay." + ); + + let event_waker = match self.event_waker.lock() { + Ok(waker_slot) => waker_slot.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }; + if let Some(event_waker) = event_waker { + event_waker(); + } + event } @@ -100,21 +143,49 @@ impl SharedScrollInputState { after_seq: u64, through: Instant, ) -> Vec<(u64, Instant, f64, f64, f64, bool, bool)> { - let queue_state = match self.queue_state.lock() { + let mut queue_state = match self.queue_state.lock() { Ok(queue_state) => queue_state, Err(poisoned) => poisoned.into_inner(), }; + let mut pruned_events = 0_usize; - queue_state + while queue_state.queue.front().is_some_and(|event| event.seq <= after_seq) { + let _ = queue_state.queue.pop_front(); + pruned_events = pruned_events.saturating_add(1); + } + + let queued_after_seq = queue_state.queue.len(); + let future_events = + queue_state.queue.iter().filter(|event| event.recorded_at > through).count(); + let replay = queue_state .queue .iter() .copied() - .filter(|event| event.seq > after_seq && event.recorded_at <= through) + .filter(|event| event.recorded_at <= through) .map(SharedScrollInputEvent::tuple) - .collect() + .collect::>(); + + if !replay.is_empty() || future_events > 0 || pruned_events > 0 { + let newest_seq = queue_state.queue.back().map(|event| event.seq).unwrap_or(0); + + tracing::info!( + op = "scroll_input.replay_window", + after_seq, + pruned_events, + queued_after_seq, + replay_count = replay.len(), + future_events, + newest_seq, + "Evaluated queued native scroll input events for overlay replay." + ); + } + + replay } } +type SharedScrollInputEventWaker = Arc; + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(in crate::app) enum ScrollInputObserverStatus { #[default] @@ -201,7 +272,6 @@ impl ScrollInputObserverLifecycle { self.set_status(ScrollInputObserverStatus::Failed); } - #[cfg(test)] pub(in crate::app) fn status(&self) -> ScrollInputObserverStatus { let status = match self.status.lock() { Ok(status) => status, @@ -255,6 +325,10 @@ struct SharedScrollInputQueueState { #[cfg(test)] mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use std::thread; use std::time::{Duration, Instant}; @@ -351,6 +425,43 @@ mod tests { ); } + #[test] + fn replay_after_seq_through_prunes_consumed_prefix_before_future_polls() { + let state = SharedScrollInputState::default(); + let start = Instant::now(); + + state.record_at(start, -4.0, 120.0, 140.0, true, false); + state.record_at(start + Duration::from_millis(1), -3.0, 120.0, 140.0, true, false); + state.record_at(start + Duration::from_millis(2), -2.0, 120.0, 140.0, true, false); + state.record_at(start + Duration::from_millis(3), -1.0, 120.0, 140.0, false, true); + + let _ = state.replay_after_seq_through(0, start + Duration::from_millis(2)); + let _ = state.replay_after_seq_through(2, start + Duration::from_millis(2)); + + let queue_state = state.queue_state.lock().unwrap(); + let queued_seqs = queue_state.queue.iter().map(|event| event.seq).collect::>(); + + assert_eq!(queued_seqs, vec![3, 4]); + } + + #[test] + fn record_invokes_event_waker() { + let state = SharedScrollInputState::default(); + let wake_count = Arc::new(AtomicUsize::new(0)); + + state.set_event_waker(Some(Arc::new({ + let wake_count = Arc::clone(&wake_count); + + move || { + wake_count.fetch_add(1, Ordering::AcqRel); + } + }))); + + state.record(-4.0, 120.0, 140.0, true, false); + + assert_eq!(wake_count.load(Ordering::Acquire), 1); + } + #[test] fn observer_lifecycle_waits_for_ready() { let lifecycle = std::sync::Arc::new(ScrollInputObserverLifecycle::default()); diff --git a/apps/rsnap/src/app/scroll_input_macos/tap.rs b/apps/rsnap/src/app/scroll_input_macos/tap.rs index 58c14ae1..0dede860 100644 --- a/apps/rsnap/src/app/scroll_input_macos/tap.rs +++ b/apps/rsnap/src/app/scroll_input_macos/tap.rs @@ -173,13 +173,33 @@ unsafe extern "C" fn scroll_input_event_tap_callback( fn send_overlay_scroll_input(context: &ScrollInputTapContext, cg_event: CGEventRef) { if !context.shared_state.is_enabled() { + tracing::info!( + op = "scroll_input.tap_ignored_disabled", + "Discarded native scroll input event because scroll capture replay is disabled." + ); + return; } let Some(decoded) = decode::decode_scroll_input_from_cg_event(cg_event) else { + tracing::info!( + op = "scroll_input.tap_ignored_decode_none", + "Discarded native scroll input event because it decoded to no usable delta." + ); + return; }; + tracing::info!( + op = "scroll_input.tap_forwarding", + delta_y = decoded.delta_y, + global_x = decoded.global_x, + global_y = decoded.global_y, + gesture_active = decoded.gesture_active, + gesture_ended = decoded.gesture_ended, + "Forwarding decoded native scroll input event into shared replay state." + ); + context.shared_state.record( decoded.delta_y, decoded.global_x, diff --git a/docs/guide/performance-checks.md b/docs/guide/performance-checks.md index 34fa7a74..ef807531 100644 --- a/docs/guide/performance-checks.md +++ b/docs/guide/performance-checks.md @@ -1,11 +1,12 @@ # Performance Checks Guide -Goal: Explain which repo-native performance command to run, when to use local deterministic -benchmarks versus dedicated macOS GUI smoke, and how to save or compare local baselines. +Goal: Explain which repo-native command to run for deterministic replay, local performance +benchmarks, or the remaining dedicated macOS GUI smoke, and how to save or compare local +baselines without confusing non-live evidence with the final live acceptance gate. -Read this when: You are investigating a performance regression, refreshing local benchmark -baselines, or deciding whether a change needs deterministic benches, dedicated desktop smoke, or -both. +Read this when: You are investigating a scroll-capture correctness or performance regression, +refreshing local benchmark baselines, or deciding whether a change needs deterministic replay, +deterministic benches, dedicated desktop smoke, or some combination of those surfaces. Inputs: `Makefile.toml`; `docs/spec/performance_tracking.md`; `docs/guide/scroll-capture-benchmarks.md` @@ -18,6 +19,12 @@ baseline workflow for the committed Criterion benchmark targets. Use the smallest command that matches the regression surface: +- Scroll-capture correctness or stitching-behavior regressions before final live validation: + `cargo make replay-scroll-capture` +- Replay-harness sanity check when no user-recorded trace is available yet: + `cargo make replay-scroll-capture-self-check` +- Scroll-capture semantic trace analysis (first bad frame, under-consumption, overshoot): + `cargo make analyze-scroll-capture-trace` - Component render regressions in egui-heavy UI such as the settings window: `cargo make perf-bench-settings-window` - Scroll-capture or image-processing hot-path regressions: @@ -29,6 +36,13 @@ Use the smallest command that matches the regression surface: - Dedicated macOS end-to-end GUI performance smoke on a logged-in desktop session: `cargo make perf-macos` +`cargo make replay-scroll-capture` and `cargo make analyze-scroll-capture-trace` +force `scroll_capture_replay --force-worker-pairwise`, so the repo-native +non-live entrypoints exercise the same worker screenshot + pairwise +registration commit path that current macOS production uses. Invoke the example +directly without that flag only when you intentionally want to compare the +legacy recorded-source replay mode. + ## What each high-level task does - `perf-local` @@ -36,22 +50,35 @@ Use the smallest command that matches the regression surface: - Use this for routine local comparisons and for regressions that do not require a real desktop session. - `perf-self-check-macos` - - Runs `perf-local`, then runs the existing macOS smoke scripts in `--self-check` mode. + - Runs `perf-local`, then runs the live-loupe self-check plus recorded-live-trace + scroll-capture replay. - Use this to validate that the dedicated macOS environment, permissions, and smoke harness are ready without treating it as an end-to-end performance assertion. - `perf-macos` - - Runs `perf-local`, then runs the real macOS GUI smoke tasks. + - Runs `perf-local`, then runs the real live-loupe GUI smoke task plus recorded-live-trace + scroll-capture replay. - Use this only on a dedicated logged-in macOS desktop session with the expected Screen Recording and automation permissions. -The low-level smoke tasks remain available: +For the downward scroll-capture rebuild, the expected verification sequence is: + +1. `cargo make check` +2. `cargo make replay-scroll-capture` +3. `cargo make analyze-scroll-capture-trace` +4. any targeted deterministic `cargo test -p rsnap-overlay ...` +5. one fresh release live touchpad run with a newly recorded trace + +The low-level deterministic and smoke tasks remain available: +- `cargo make replay-scroll-capture` +- `cargo make replay-scroll-capture-self-check` +- `cargo make analyze-scroll-capture-trace` - `cargo make smoke-live-loupe-perf-macos` -- `cargo make smoke-scroll-capture-macos` - `cargo make smoke-self-check-macos` - `cargo make smoke-macos` -Use them when you need to isolate one smoke harness instead of the high-level performance entrypoint. +Use them when you need to isolate deterministic scroll-capture replay or the live-loupe smoke +harness instead of the high-level performance entrypoint. ## Baseline workflow for local benchmarks @@ -82,6 +109,8 @@ Dedicated macOS smoke: - Requires a logged-in macOS desktop session. - Requires the expected Screen Recording and automation permissions for the smoke scripts. +- Only covers the remaining live-loupe desktop path; scroll-capture correctness is now exercised by + deterministic replay. - Is meant for dedicated-host or manual validation, not a flaky shared-runner PR gate. ## Interpreting failures @@ -89,6 +118,17 @@ Dedicated macOS smoke: - `perf-bench-settings-window` or `perf-bench-scroll-capture` regressions: compare scenario-level numbers against your saved baseline and inspect the relevant benchmark group before escalating to GUI smoke. +- `replay-scroll-capture` failures: + treat them as authoritative regressions against the latest recorded live trace in shipping + worker-pairwise overlay or session logic before attempting more desktop-session repro. If the + command reports that no trace manifests were found, that is an operator/setup failure: record a + fresh live trace first or rerun the example with `--trace `. +- `replay-scroll-capture-self-check` failures: + treat them as deterministic regressions in the replay harness itself, not as evidence about the + latest user-recorded live trace. +- clean replay plus trace analysis: + this is necessary but not sufficient for XY-185 style sign-off; the remaining risk is + isolated to the final fresh live touchpad run, not eliminated. - `perf-self-check-macos` failures: treat these first as environment or permission readiness failures unless local benches also regressed. diff --git a/docs/guide/scroll-capture-benchmarks.md b/docs/guide/scroll-capture-benchmarks.md index 57542dc3..cdae2774 100644 --- a/docs/guide/scroll-capture-benchmarks.md +++ b/docs/guide/scroll-capture-benchmarks.md @@ -14,6 +14,31 @@ Depends on: `docs/spec/performance_tracking.md` Outputs: A repeatable local benchmark run, an optional saved Criterion baseline, and a clear understanding of what the synthetic fixture is intended to cover. +If you are debugging correctness rather than hot-path speed, start with: + +```bash +cargo make replay-scroll-capture +cargo make replay-scroll-capture-self-check +``` + +That replay runner exercises shipping overlay and session logic against the latest recorded live +trace and should be treated as the primary non-live correctness surface. The repo-native replay +tasks force the worker-pairwise mode so they match current macOS production scroll-capture +authority. It requires a recorded manifest under `~/Library/Application Support/ink.hack.rsnap/scroll-capture-traces/` +unless you invoke the example with `--trace `. Use the direct example without +`--force-worker-pairwise` only when you intentionally want to compare the legacy recorded-source +replay path. `cargo make replay-scroll-capture-self-check` is the deterministic fallback when you +want to validate the replay harness without depending on a user-recorded trace. For semantic analysis +(first bad frame, under-consumption, overshoot), use: + +```bash +cargo make analyze-scroll-capture-trace +``` + +Use the benchmark target below only when you specifically need performance numbers. A clean +benchmark run does not replace replay, trace analysis, or the final fresh live touchpad +acceptance run. + ## Fixture contract The committed benchmark fixture is code-generated inside `scroll_capture::bench_support`; it does diff --git a/docs/spec/v0.md b/docs/spec/v0.md index 524c310b..f0535309 100644 --- a/docs/spec/v0.md +++ b/docs/spec/v0.md @@ -82,20 +82,28 @@ cross-platform architecture. scroll capture, even if a frozen image exists. - The frozen toolbar may expose `Scroll Capture ↓`, and plain `s` may start scroll capture, whenever the frozen capture source is a dragged region on macOS. - - `SCStream` remains the image source. Captured-frame overlap, viewport reacquisition, - and committed crop proof are the source of truth for scroll progress and rewind - recovery. - - Stitching is downward-only. Down-scroll may refresh preview state and append - committed rows. Up-scroll / rewind may be observed, but it must never append stitched - growth. + - The image source for scroll-capture commits is discrete monitor-region screenshots + captured through the native macOS screenshot API. Live streams may still support the + surrounding overlay experience, but they are not the geometry authority for downward + stitch commits. + - Pairwise image registration plus overlap proof between adjacent discrete screenshots is + the source of truth for downward scroll progress, viewport reacquisition, and append + eligibility. + - Stitching is downward-only. Down-scroll may refresh preview state and append committed + rows. Up-scroll / rewind may be observed, but it must never append stitched growth. - After an upward rewind, growth stays blocked until captured-frame proof reacquires the last committed viewport and then re-advances past the resume frontier. Do not resume from stale, weak, or best-guess heuristics. - - If crop proof is weak, ambiguous, stale, or otherwise not trustworthy, fail closed: - no append, no position advance, and no best-guess resume. Keep the session blocked - or preview-only until trustworthy proof arrives. - - `Space` copies the stitched image and exits. Cmd+S (macOS) / Ctrl+S saves it and - exits. `Esc` / `Back` stops scroll capture and restores the original Frozen capture. + - If pairwise proof is weak, ambiguous, stale, or otherwise not trustworthy, fail closed: + no append, no position advance, and no best-guess resume. Keep the session blocked or + preview-only until trustworthy proof arrives. + - Preview, `Space`, and save/export must all render from the same committed stitched + canvas. Provisional or preview-only state must never produce a different clipboard or + saved result from what the user sees. + - `Space` copies the stitched image and exits. Cmd+S (macOS) / Ctrl+S saves it and exits. + `Esc` / `Back` stops scroll capture and restores the original Frozen capture. + - Verification order is part of the contract: deterministic and replay entrypoints must + pass before any final live touchpad acceptance run is treated as authoritative. ## HUD, blur, tint and hue controls diff --git a/packages/rsnap-overlay/Cargo.toml b/packages/rsnap-overlay/Cargo.toml index d9c96fa2..9ca3c4ee 100644 --- a/packages/rsnap-overlay/Cargo.toml +++ b/packages/rsnap-overlay/Cargo.toml @@ -18,6 +18,7 @@ cargo-clippy = [] [dependencies] arboard = { workspace = true } color-eyre = { workspace = true } +directories = { workspace = true } egui = { workspace = true } egui-phosphor = { workspace = true } egui-wgpu = { workspace = true } @@ -25,8 +26,10 @@ egui-winit = { workspace = true } image = { workspace = true } pollster = { workspace = true } serde = { workspace = true } +serde_json = "1.0" thiserror = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } wgpu = { workspace = true } winit = { workspace = true } @@ -36,6 +39,7 @@ xcap = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] block2 = { workspace = true } +dispatch2 = "0.3" objc = { workspace = true } objc2 = { workspace = true } objc2-app-kit = { workspace = true } @@ -45,6 +49,7 @@ objc2-core-media = { workspace = true } objc2-core-video = { workspace = true } objc2-foundation = { workspace = true } objc2-screen-capture-kit = { workspace = true } +objc2-vision = "0.3.2" raw-window-handle = { workspace = true } [dev-dependencies] diff --git a/packages/rsnap-overlay/examples/scroll_capture_replay.rs b/packages/rsnap-overlay/examples/scroll_capture_replay.rs new file mode 100644 index 00000000..63830b37 --- /dev/null +++ b/packages/rsnap-overlay/examples/scroll_capture_replay.rs @@ -0,0 +1,181 @@ +//! Deterministic scroll-capture replay runner used by cargo-make verification tasks. + +#![allow(unused_crate_dependencies)] + +use std::path::PathBuf; +use std::process::ExitCode; + +use color_eyre::eyre::WrapErr; +use directories::ProjectDirs; +use rsnap_overlay::replay_support::{ + RecordedScrollCaptureReplayMode, replay_recorded_scroll_capture_trace_with_mode, +}; +use tracing_subscriber::{EnvFilter, fmt}; + +fn main() -> ExitCode { + if let Err(err) = run() { + eprintln!("[replay] {err}"); + + return ExitCode::FAILURE; + } + + ExitCode::SUCCESS +} + +fn run() -> color_eyre::Result<()> { + color_eyre::install()?; + let _ = fmt() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("warn,rsnap_overlay=info")), + ) + .with_target(false) + .with_level(true) + .try_init(); + + let mut args = std::env::args().skip(1); + let mut trace_manifest_path = None; + let mut list_only = false; + let mut emit_json = false; + let mut summary_only = false; + let mut force_worker_pairwise = false; + + while let Some(arg) = args.next() { + match arg.as_str() { + "--list" | "--self-check" => { + list_only = true; + }, + "--json" => { + emit_json = true; + }, + "--summary-only" => { + summary_only = true; + }, + "--force-worker-pairwise" => { + force_worker_pairwise = true; + }, + "--trace" => { + let Some(value) = args.next() else { + color_eyre::eyre::bail!("--trace requires a manifest path"); + }; + trace_manifest_path = Some(value); + }, + other => { + color_eyre::eyre::bail!( + "unknown argument {other}; supported flags are --list, --self-check, --json, --summary-only, --force-worker-pairwise, and --trace " + ); + }, + } + } + + if list_only { + println!("latest-recorded-live-trace"); + + return Ok(()); + } + + let trace_manifest_path = trace_manifest_path.map(Ok).unwrap_or_else(|| { + latest_recorded_trace_manifest().map(|path| path.display().to_string()) + })?; + let replay_mode = if force_worker_pairwise { + RecordedScrollCaptureReplayMode::ForceWorkerPairwise + } else { + RecordedScrollCaptureReplayMode::RecordedSource + }; + let mut summary = + replay_recorded_scroll_capture_trace_with_mode(&trace_manifest_path, replay_mode)?; + if summary_only { + summary.step_results.clear(); + } + if emit_json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else { + print_recorded_trace_summary(&summary); + } + + Ok(()) +} + +fn print_recorded_trace_summary( + summary: &rsnap_overlay::replay_support::RecordedScrollCaptureReplaySummary, +) { + println!( + "[replay] mode={:?} trace={} manifest={} final_export_height={} final_preview_height={} final_viewport_top_y={} recorded_final_export_height={:?} recorded_final_preview_height={:?} first_outcome_divergence_frame={:?} first_export_height_drift_frame={:?} first_preview_height_drift_frame={:?} first_semantic_issue_frame={:?} first_missed_downward_motion_frame={:?} first_underconsumed_downward_motion_frame={:?} first_growth_overshoot_frame={:?} max_recorded_committed_growth_rows={} max_replayed_committed_growth_rows={} max_recorded_export_jump={} max_recorded_preview_jump={} max_replayed_export_jump={} max_replayed_preview_jump={} final_preview_path={:?} final_export_path={:?}", + summary.replay_mode, + summary.trace_id, + summary.manifest_path.display(), + summary.final_export_height, + summary.final_preview_height, + summary.final_viewport_top_y, + summary.recorded_final_export_height, + summary.recorded_final_preview_height, + summary.first_outcome_divergence_frame, + summary.first_export_height_drift_frame, + summary.first_preview_height_drift_frame, + summary.first_semantic_issue_frame, + summary.first_missed_downward_motion_frame, + summary.first_underconsumed_downward_motion_frame, + summary.first_growth_overshoot_frame, + summary.max_recorded_committed_growth_rows, + summary.max_replayed_committed_growth_rows, + summary.max_recorded_export_jump, + summary.max_recorded_preview_jump, + summary.max_replayed_export_jump, + summary.max_replayed_preview_jump, + summary.final_preview_path, + summary.final_export_path + ); + + for step in &summary.step_results { + println!( + "[replay] frame={} path={} observed_at_ms={} source={:?} live_frame_gap={:?} recorded={:?} replayed={:?} estimated_shift={:?} semantic_issue={:?} export_height={} preview_height={} recorded_export_height={:?} recorded_preview_height={:?} viewport_top_y={} last_commit_source={:?} last_commit_motion_rows={:?} last_block_reason={:?} replayed_registration_result={:?} replayed_registration_source={:?} replayed_registration_motion_rows={:?} replayed_candidates_before={:?} replayed_candidates_after={:?} replayed_last_hint={:?} replayed_transient_hint={:?} replayed_effective_hint={:?} replayed_burst={} replayed_preview_local_top={:?}", + step.frame_index, + step.frame_path, + step.observed_at_ms, + step.frame_source, + step.live_frame_gap, + step.recorded_outcome, + step.replayed_outcome, + step.recorded_estimated_downward_shift_rows, + step.semantic_issue, + step.export_height, + step.preview_height, + step.recorded_export_height, + step.recorded_preview_height, + step.viewport_top_y, + step.last_commit_decision_source, + step.last_commit_detected_motion_rows, + step.last_block_reason, + step.replayed_downward_sample_registration_result, + step.replayed_downward_sample_registration_source, + step.replayed_downward_sample_registration_motion_rows, + step.replayed_downward_viewport_candidates_before_prune, + step.replayed_downward_viewport_candidates_after_prune, + step.replayed_sample_eval_last_motion_rows_hint, + step.replayed_sample_eval_transient_motion_rows_hint, + step.replayed_sample_eval_effective_motion_rows_hint, + step.replayed_sample_eval_transient_burst_search_enabled, + step.replayed_preview_only_local_viewport_top_y + ); + } +} + +fn latest_recorded_trace_manifest() -> color_eyre::Result { + let project_dirs = ProjectDirs::from("ink", "hack", "rsnap") + .expect("rsnap project directories should be available"); + let trace_root = project_dirs.data_local_dir().join("scroll-capture-traces"); + let mut manifests: Vec = std::fs::read_dir(&trace_root) + .wrap_err_with(|| format!("failed to read {}", trace_root.display()))? + .filter_map(Result::ok) + .map(|entry| entry.path().join("manifest.json")) + .filter(|path| path.exists()) + .collect(); + + manifests.sort(); + manifests.pop().ok_or_else(|| { + color_eyre::eyre::eyre!( + "no recorded scroll-capture trace manifests found under {}; record a fresh live trace first or pass --trace ", + trace_root.display() + ) + }) +} diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index 09c4c54e..c14de813 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -13,18 +13,28 @@ use std::process; use std::ptr; use std::sync::Arc; #[cfg(target_os = "macos")] +use std::sync::{Mutex, mpsc}; +#[cfg(target_os = "macos")] use std::thread; use std::time::{Duration, Instant}; +#[cfg(target_os = "macos")] +use block2::RcBlock; use color_eyre::eyre::{self, Result, WrapErr}; use image::RgbaImage; use image::imageops; #[cfg(target_os = "macos")] +use objc2::rc::Retained; +#[cfg(target_os = "macos")] use objc2_core_foundation::{CGPoint, CGRect, CGSize}; #[cfg(target_os = "macos")] use objc2_core_graphics::{ CGDataProvider, CGImage, CGRectNull, CGWindowID, CGWindowImageOption, CGWindowListOption, }; +#[cfg(target_os = "macos")] +use objc2_foundation::NSError; +#[cfg(target_os = "macos")] +use objc2_screen_capture_kit::SCScreenshotManager; use thiserror::Error; #[cfg(not(target_os = "macos"))] use xcap::Window; @@ -76,6 +86,14 @@ const K_CF_NUMBER_CG_FLOAT_TYPE: u32 = 16; const MACOS_REGION_FRAME_WAIT_TIMEOUT: Duration = Duration::from_millis(120); #[cfg(target_os = "macos")] const MACOS_REGION_FRAME_WAIT_POLL_INTERVAL: Duration = Duration::from_millis(8); +#[cfg(target_os = "macos")] +const MACOS_SCREENSHOT_CAPTURE_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(target_os = "macos")] +const MACOS_SCREENSHOT_ERROR_TIMEOUT_CODE: isize = 10; +#[cfg(target_os = "macos")] +const MACOS_SCREENSHOT_ERROR_NULL_IMAGE_CODE: isize = 11; +#[cfg(target_os = "macos")] +const MACOS_SCREENSHOT_ERROR_RETAIN_FAILED_CODE: isize = 12; /// Capture backend contract used by the overlay worker. pub trait CaptureBackend: Send { @@ -503,34 +521,20 @@ impl XcapCaptureBackend { monitor: MonitorRect, rect_px: RectPoints, ) -> Result> { - let after_frame_seq = self.region_capture_after_seq(monitor, rect_px); - - if let Some((frame_seq, image)) = - self.wait_for_live_stream_region(monitor, rect_px, after_frame_seq) - { - self.record_region_capture(monitor, rect_px, frame_seq); - - tracing::trace!( - op = "capture_backend.region_stream_hit", - monitor_id = monitor.id, - rect_px = ?rect_px, - frame_seq, - frame_px = ?image.dimensions(), - "Captured monitor region from ScreenCaptureKit stream." - ); - - return Ok(Some(image)); - } + let image = capture_monitor_region_image_with_screenshot_manager(monitor, rect_px) + .wrap_err_with(|| { + format!("failed to capture monitor region via SCScreenshotManager: {monitor:?}") + })?; tracing::trace!( - op = "capture_backend.region_stream_no_new_frame_for_scroll_capture", + op = "capture_backend.region_screenshot_hit", monitor_id = monitor.id, rect_px = ?rect_px, - after_frame_seq, - "Did not receive a fresh ScreenCaptureKit region frame for scroll capture." + frame_px = ?image.dimensions(), + "Captured monitor region from ScreenCaptureKit screenshot API." ); - Ok(None) + Ok(Some(image)) } #[cfg(target_os = "macos")] @@ -957,10 +961,39 @@ fn rgba_image_from_cg_image(cg_image: &CGImage) -> Result { let data = CGDataProvider::data(Some(data_provider.as_ref())) .ok_or_else(|| eyre::eyre!("Failed to copy CGImage bytes"))?; let bytes_per_row = CGImage::bytes_per_row(Some(cg_image)); - let mut buffer = Vec::with_capacity(width * height * 4); + rgba_image_from_bgra_rows(width, height, bytes_per_row, &data.to_vec()) +} - for row in data.to_vec().chunks_exact(bytes_per_row) { - buffer.extend_from_slice(&row[..width * 4]); +#[cfg(target_os = "macos")] +fn rgba_image_from_bgra_rows( + width: usize, + height: usize, + bytes_per_row: usize, + data: &[u8], +) -> Result { + let expected_row_bytes = + width.checked_mul(4).ok_or_else(|| eyre::eyre!("row byte count overflowed"))?; + + if bytes_per_row < expected_row_bytes { + return Err(eyre::eyre!( + "CGImage bytes_per_row {bytes_per_row} is smaller than the required RGBA row width {expected_row_bytes}" + )); + } + + let required_len = height + .checked_mul(bytes_per_row) + .ok_or_else(|| eyre::eyre!("CGImage backing store length overflowed"))?; + + if data.len() < required_len { + return Err(eyre::eyre!( + "CGImage backing store is shorter than the declared image size: expected at least {required_len} bytes, got {}", + data.len() + )); + } + + let mut buffer = Vec::with_capacity(width * height * 4); + for row in data[..required_len].chunks_exact(bytes_per_row) { + buffer.extend_from_slice(&row[..expected_row_bytes]); } for bgra in buffer.chunks_exact_mut(4) { bgra.swap(0, 2); @@ -1059,6 +1092,71 @@ fn crop_monitor_image_region(image: &RgbaImage, rect_px: RectPoints) -> Result Result { + let rect_px = normalize_capture_rect(rect_px); + let sf = f64::from(monitor.scale_factor()).max(1.0); + let cg_rect = CGRect::new( + CGPoint::new( + f64::from(monitor.origin.x) + f64::from(rect_px.x) / sf, + f64::from(monitor.origin.y) + f64::from(rect_px.y) / sf, + ), + CGSize::new(f64::from(rect_px.width) / sf, f64::from(rect_px.height) / sf), + ); + let cg_image = capture_screenshot_cg_image(cg_rect)?; + let image = rgba_image_from_cg_image(cg_image.as_ref())?; + + // ScreenCaptureKit may round point-space captures by one pixel at non-integer scale edges. + // Clamp back to the requested region so the stitcher sees stable dimensions. + if image.dimensions() == (rect_px.width, rect_px.height) { + Ok(image) + } else { + crop_monitor_image_region(&image, RectPoints::new(0, 0, rect_px.width, rect_px.height)) + } +} + +#[cfg(target_os = "macos")] +fn capture_screenshot_cg_image(rect: CGRect) -> Result> { + let (tx, rx) = mpsc::sync_channel::, Retained>>(1); + let tx = Mutex::new(Some(tx)); + let block = RcBlock::new(move |image: *mut CGImage, err: *mut NSError| { + let mut maybe_tx = match tx.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let Some(tx) = maybe_tx.take() else { + return; + }; + + if !err.is_null() { + let Some(err) = (unsafe { Retained::retain(err) }) else { + let _ = tx.send(Err(screenshot_error(MACOS_SCREENSHOT_ERROR_RETAIN_FAILED_CODE))); + + return; + }; + let _ = tx.send(Err(err)); + + return; + } + + let Some(image) = (unsafe { Retained::retain(image) }) else { + let _ = tx.send(Err(screenshot_error(MACOS_SCREENSHOT_ERROR_NULL_IMAGE_CODE))); + + return; + }; + let _ = tx.send(Ok(image)); + }); + + unsafe { SCScreenshotManager::captureImageInRect_completionHandler(rect, Some(&block)) }; + + rx.recv_timeout(MACOS_SCREENSHOT_CAPTURE_TIMEOUT) + .map_err(|_| screenshot_error(MACOS_SCREENSHOT_ERROR_TIMEOUT_CODE))? + .map_err(|err| eyre::eyre!("{}", err.localizedDescription())) +} + #[cfg(target_os = "macos")] fn collect_window_geometries() -> Result> { let window_list_ref = unsafe { @@ -1232,6 +1330,11 @@ fn cf_dictionary_at_index(array: CFArrayRef, index: isize) -> Option Retained { + NSError::new(code, objc2_foundation::ns_string!("io.hackink.rsnap.screenshot_capture")) +} + #[cfg(not(target_os = "macos"))] fn collect_window_geometries() -> Result> { let windows = Window::all().wrap_err("xcap Window::all failed")?; @@ -1367,4 +1470,42 @@ mod tests { assert_eq!(backend.region_capture_after_seq(monitor, other_rect), 0); assert_eq!(backend.region_capture_after_seq(other_monitor, rect), 0); } + + #[cfg(target_os = "macos")] + #[test] + fn rgba_image_from_bgra_rows_truncates_trailing_rows_beyond_declared_height() { + let width = 2_usize; + let height = 2_usize; + let bytes_per_row = width * 4; + let data = vec![ + 10, 20, 30, 255, 40, 50, 60, 255, // row 0 + 70, 80, 90, 255, 100, 110, 120, 255, // row 1 + 130, 140, 150, 255, 160, 170, 180, 255, // extra row 2 + 190, 200, 210, 255, 220, 230, 240, 255, // extra row 3 + ]; + + let image = crate::backend::rgba_image_from_bgra_rows(width, height, bytes_per_row, &data) + .expect("image should decode"); + + assert_eq!(image.dimensions(), (2, 2)); + assert_eq!(image.as_raw().len(), width * height * 4); + assert_eq!(image.get_pixel(0, 0).0, [30, 20, 10, 255]); + assert_eq!(image.get_pixel(1, 1).0, [120, 110, 100, 255]); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgba_image_from_bgra_rows_rejects_short_backing_store() { + let err = crate::backend::rgba_image_from_bgra_rows( + 2, + 2, + 8, + &[ + 10, 20, 30, 255, 40, 50, 60, 255, // row 0 + ], + ) + .expect_err("short backing store should fail"); + + assert!(format!("{err:#}").contains("shorter than the declared image size")); + } } diff --git a/packages/rsnap-overlay/src/lib.rs b/packages/rsnap-overlay/src/lib.rs index 74f29669..1a8b2d38 100644 --- a/packages/rsnap-overlay/src/lib.rs +++ b/packages/rsnap-overlay/src/lib.rs @@ -13,6 +13,15 @@ pub mod bench_support { }; } +/// Deterministic replay harness exports for scroll-capture verification. +pub mod replay_support { + pub use crate::overlay::replay_support::{ + RecordedScrollCaptureReplayMode, RecordedScrollCaptureReplayRecordedOutcome, + RecordedScrollCaptureReplayStepResult, RecordedScrollCaptureReplaySummary, + replay_recorded_scroll_capture_trace, replay_recorded_scroll_capture_trace_with_mode, + }; +} + mod backend; #[cfg(target_os = "macos")] mod live_frame_stream_macos; diff --git a/packages/rsnap-overlay/src/live_frame_stream_macos.rs b/packages/rsnap-overlay/src/live_frame_stream_macos.rs index d20fc965..b302b45f 100644 --- a/packages/rsnap-overlay/src/live_frame_stream_macos.rs +++ b/packages/rsnap-overlay/src/live_frame_stream_macos.rs @@ -16,11 +16,12 @@ use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use block2::RcBlock; +use dispatch2::{DispatchQueue, DispatchQueueAttr, DispatchRetained}; use image::RgbaImage; use objc2::rc::Retained; use objc2::runtime::ProtocolObject; use objc2::{AnyThread, DefinedClass, Message}; -use objc2_core_foundation::CFRetained; +use objc2_core_foundation::{CFRetained, CGPoint, CGRect, CGSize}; use objc2_core_media::{CMSampleBuffer, kCMTimeZero}; use objc2_core_video::{ CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBytesPerRow, @@ -89,7 +90,18 @@ objc2::define_class!( } frames.push_back(frame.clone()); drop(frames); - self.ivars().shared_latest_frame.store(self.ivars().monitor_id, &frame); + let store_outcome = + self.ivars().shared_latest_frame.store(self.ivars().monitor_id, &frame); + if store_outcome.completed_ensure || store_outcome.completed_refresh { + tracing::info!( + op = "live_frame_stream.frame_received", + monitor_id = self.ivars().monitor_id, + frame_seq, + completed_ensure = store_outcome.completed_ensure, + completed_refresh = store_outcome.completed_refresh, + "Received a ScreenCaptureKit frame that satisfied a pending ensure or refresh request." + ); + } if let Some(frame_waker) = self.ivars().frame_waker.as_ref() { frame_waker(); @@ -101,14 +113,29 @@ objc2::define_class!( const STREAM_RPC_TIMEOUT: Duration = Duration::from_secs(3); const STREAM_SETUP_BACKOFF: Duration = Duration::from_millis(300); const STREAM_INCOMPLETE_EXCEPTION_UPGRADE_BACKOFF: Duration = Duration::from_secs(3); -const STREAM_FRAME_QUEUE_CAPACITY: usize = 8; +const STREAM_FRAME_QUEUE_CAPACITY: usize = 16; +const STREAM_CONFIG_QUEUE_DEPTH: usize = 8; const STREAM_REGION_FRAME_MAX_AGE: Duration = Duration::from_millis(90); +const STREAM_ACTIVE_GESTURE_FORCE_REFRESH_MIN_AGE: Duration = Duration::from_millis(60); const STREAM_REGION_FRAME_REFRESH_TIMEOUT: Duration = Duration::from_millis(180); const STREAM_REGION_FRAME_REFRESH_POLL_INTERVAL: Duration = Duration::from_millis(4); +const STREAM_POST_SETUP_FRAME_GRACE: Duration = STREAM_SETUP_BACKOFF; const STREAM_ERROR_TIMEOUT_CODE: isize = 1; const STREAM_ERROR_NULL_CONTENT_CODE: isize = 2; const STREAM_ERROR_RETAIN_FAILED_CODE: isize = 3; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct StreamCaptureRegion { + rect_points: RectPoints, + rect_pixels: RectPoints, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamCaptureTarget { + FullMonitor, + Region(StreamCaptureRegion), +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum StreamFilterMode { ExcludeCurrentProcess, @@ -119,6 +146,9 @@ enum WorkerRequest { EnsureMonitor { monitor: MonitorRect, }, + RefreshMonitor { + monitor: MonitorRect, + }, SampleCursor { monitor: MonitorRect, x_px: u32, @@ -179,20 +209,28 @@ impl CursorSampleRequest { pub(crate) struct MacLiveFrameStream { request_tx: Sender, shared_latest_frame: Arc, + capture_target: StreamCaptureTarget, worker: Option>, #[cfg(test)] debug_filter: StreamFilterConfig, + #[cfg(test)] + debug_last_request_kind: Arc>>, } impl MacLiveFrameStream { pub(crate) fn new() -> Self { - Self::with_self_capture_exception_window_ids(Vec::new()) + Self::with_capture_target_and_filter_and_waker( + StreamCaptureTarget::FullMonitor, + StreamFilterConfig { self_capture_exception_window_ids: Vec::new() }, + None, + ) } pub(crate) fn with_self_capture_exception_window_ids( self_capture_exception_window_ids: Vec, ) -> Self { - Self::with_self_capture_exception_window_ids_and_waker( - self_capture_exception_window_ids, + Self::with_capture_target_and_filter_and_waker( + StreamCaptureTarget::FullMonitor, + StreamFilterConfig { self_capture_exception_window_ids }, None, ) } @@ -201,36 +239,74 @@ impl MacLiveFrameStream { self_capture_exception_window_ids: Vec, frame_waker: Option>, ) -> Self { - Self::with_filter_and_waker( + Self::with_capture_target_and_filter_and_waker( + StreamCaptureTarget::FullMonitor, StreamFilterConfig { self_capture_exception_window_ids }, frame_waker, ) } + pub(crate) fn with_scroll_capture_region_and_waker( + self_capture_exception_window_ids: Vec, + rect_points: RectPoints, + rect_pixels: RectPoints, + frame_waker: Option>, + ) -> Self { + Self::with_capture_target_and_filter_and_waker( + StreamCaptureTarget::Region(StreamCaptureRegion { rect_points, rect_pixels }), + StreamFilterConfig { self_capture_exception_window_ids }, + frame_waker, + ) + } + + fn with_capture_target_and_filter_and_waker( + capture_target: StreamCaptureTarget, + filter: StreamFilterConfig, + frame_waker: Option>, + ) -> Self { + Self::with_filter_and_waker(capture_target, filter, frame_waker) + } + fn with_filter_and_waker( + capture_target: StreamCaptureTarget, filter: StreamFilterConfig, frame_waker: Option>, ) -> Self { #[cfg(test)] let debug_filter = filter.clone(); + #[cfg(test)] + let debug_last_request_kind = Arc::new(Mutex::new(None)); let (request_tx, request_rx) = mpsc::channel(); let shared_latest_frame = Arc::new(SharedLatestFrame::default()); let worker_shared_latest_frame = shared_latest_frame.clone(); let worker = thread::spawn(move || { - stream_worker_loop(request_rx, frame_waker, worker_shared_latest_frame, filter); + stream_worker_loop( + request_rx, + frame_waker, + worker_shared_latest_frame, + filter, + capture_target, + ); }); Self { request_tx, shared_latest_frame, + capture_target, worker: Some(worker), #[cfg(test)] debug_filter, + #[cfg(test)] + debug_last_request_kind, } } pub(crate) fn with_waker(frame_waker: Option>) -> Self { - Self::with_self_capture_exception_window_ids_and_waker(Vec::new(), frame_waker) + Self::with_capture_target_and_filter_and_waker( + StreamCaptureTarget::FullMonitor, + StreamFilterConfig { self_capture_exception_window_ids: Vec::new() }, + frame_waker, + ) } #[cfg(test)] @@ -238,6 +314,27 @@ impl MacLiveFrameStream { &self.debug_filter.self_capture_exception_window_ids } + #[cfg(test)] + pub(crate) fn debug_last_request_kind(&self) -> Option<&'static str> { + match self.debug_last_request_kind.lock() { + Ok(guard) => *guard, + Err(poisoned) => *poisoned.into_inner(), + } + } + + #[cfg(test)] + fn record_debug_request_kind(&self, kind: &'static str) { + match self.debug_last_request_kind.lock() { + Ok(mut guard) => { + *guard = Some(kind); + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + *guard = Some(kind); + }, + } + } + pub(crate) fn sample_rgb(&mut self, monitor: MonitorRect, x_px: u32, y_px: u32) -> Option { self.request(|reply_tx| WorkerRequest::SampleCursor { monitor, @@ -291,7 +388,7 @@ impl MacLiveFrameStream { }); if sample.is_none() { - self.ensure_monitor_nonblocking(monitor); + self.prime_monitor_nonblocking(monitor); } sample @@ -309,7 +406,7 @@ impl MacLiveFrameStream { monitor: MonitorRect, ) -> Option> { let Some(frame) = self.shared_latest_frame.latest_frame_for_monitor(monitor.id) else { - self.ensure_monitor_nonblocking(monitor); + self.prime_monitor_nonblocking(monitor); return None; }; @@ -328,6 +425,9 @@ impl MacLiveFrameStream { monitor: MonitorRect, rect_px: RectPoints, ) -> Option { + #[cfg(test)] + self.record_debug_request_kind("latest_rgba_region"); + self.request(|reply_tx| WorkerRequest::LatestRgbaRegion { monitor, rect_px, reply_tx }) .flatten() } @@ -338,6 +438,9 @@ impl MacLiveFrameStream { rect_px: RectPoints, after_frame_seq: u64, ) -> Option<(u64, RgbaImage)> { + #[cfg(test)] + self.record_debug_request_kind("latest_rgba_region_if_new"); + let mut frames = self.ordered_rgba_regions_after_seq(monitor, rect_px, after_frame_seq)?; let frame = frames.pop()?; @@ -350,6 +453,9 @@ impl MacLiveFrameStream { rect_px: RectPoints, after_frame_seq: u64, ) -> Option> { + #[cfg(test)] + self.record_debug_request_kind("ordered_rgba_regions_after_seq"); + self.request(|reply_tx| WorkerRequest::OrderedRgbaRegionsAfterSeq { monitor, rect_px, @@ -359,6 +465,37 @@ impl MacLiveFrameStream { .flatten() } + pub(crate) fn ordered_rgba_regions_after_seq_nonblocking( + &mut self, + monitor: MonitorRect, + rect_px: RectPoints, + after_frame_seq: u64, + ) -> Option> { + #[cfg(test)] + self.record_debug_request_kind("ordered_rgba_regions_after_seq_nonblocking"); + + let frames = + self.shared_latest_frame.frames_after_seq_for_monitor(monitor.id, after_frame_seq); + if frames.is_empty() { + if self.shared_latest_frame.latest_frame_for_monitor(monitor.id).is_none() { + self.prime_monitor_nonblocking(monitor); + } + + return None; + } + + let stream_rect_px = self.stream_rect_for_requested_region(rect_px)?; + let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); + (!frames.is_empty()).then_some(frames) + } + + fn stream_rect_for_requested_region( + &self, + requested_rect_px: RectPoints, + ) -> Option { + stream_rect_for_requested_region(self.capture_target, requested_rect_px) + } + fn request(&self, build_request: impl FnOnce(Sender) -> WorkerRequest) -> Option { let (reply_tx, reply_rx) = mpsc::channel(); @@ -367,7 +504,7 @@ impl MacLiveFrameStream { reply_rx.recv_timeout(STREAM_RPC_TIMEOUT).ok() } - fn ensure_monitor_nonblocking(&self, monitor: MonitorRect) { + pub(crate) fn prime_monitor_nonblocking(&self, monitor: MonitorRect) { if !self.shared_latest_frame.begin_ensure_monitor(monitor.id) { return; } @@ -375,6 +512,94 @@ impl MacLiveFrameStream { self.shared_latest_frame.finish_ensure_monitor(monitor.id); } } + + pub(crate) fn refresh_monitor_nonblocking_if_stale( + &self, + monitor: MonitorRect, + after_frame_seq: u64, + force_refresh: bool, + ) -> bool { + #[cfg(test)] + self.record_debug_request_kind("refresh_monitor_nonblocking_if_stale"); + + let now = Instant::now(); + + if self.shared_latest_frame.waiting_for_frame_after_setup(monitor.id) { + return false; + } + let Some(latest_frame) = self.shared_latest_frame.latest_frame_for_monitor(monitor.id) + else { + self.prime_monitor_nonblocking(monitor); + + return true; + }; + let frame_age = Instant::now().saturating_duration_since(latest_frame.captured_at); + + if !should_refresh_monitor_frame( + latest_frame.frame_seq, + after_frame_seq, + frame_age, + force_refresh, + ) { + return false; + } + if !self.shared_latest_frame.begin_refresh_monitor(monitor.id, after_frame_seq, now) { + return false; + } + if self.request_tx.send(WorkerRequest::RefreshMonitor { monitor }).is_err() { + self.shared_latest_frame.finish_refresh_monitor(monitor.id); + + return false; + } + + true + } +} + +fn stream_rect_for_requested_region( + capture_target: StreamCaptureTarget, + requested_rect_px: RectPoints, +) -> Option { + match capture_target { + StreamCaptureTarget::FullMonitor => Some(requested_rect_px), + StreamCaptureTarget::Region(region) => { + let relative_x = requested_rect_px.x.checked_sub(region.rect_pixels.x)?; + let relative_y = requested_rect_px.y.checked_sub(region.rect_pixels.y)?; + let requested_right = requested_rect_px.x.checked_add(requested_rect_px.width)?; + let requested_bottom = requested_rect_px.y.checked_add(requested_rect_px.height)?; + let region_right = region.rect_pixels.x.checked_add(region.rect_pixels.width)?; + let region_bottom = region.rect_pixels.y.checked_add(region.rect_pixels.height)?; + + if requested_right > region_right || requested_bottom > region_bottom { + return None; + } + + Some(RectPoints::new( + relative_x, + relative_y, + requested_rect_px.width, + requested_rect_px.height, + )) + }, + } +} + +fn should_refresh_monitor_frame( + latest_frame_seq: u64, + after_frame_seq: u64, + frame_age: Duration, + force_refresh: bool, +) -> bool { + if latest_frame_seq > after_frame_seq { + return false; + } + + if force_refresh { + let _ = frame_age; + return true; + } + + frame_age > STREAM_REGION_FRAME_MAX_AGE } impl Drop for MacLiveFrameStream { @@ -417,41 +642,132 @@ struct QueuedPixelBufferFrame { } #[derive(Clone)] -struct LatestQueuedPixelBufferFrame { +struct SharedQueuedPixelBufferFrames { monitor_id: u32, - frame: QueuedPixelBufferFrame, + frames: VecDeque, } #[derive(Default)] struct SharedLatestFrame { - latest: Mutex>, + frames: Mutex>, pending_monitor: Mutex>, + pending_refresh_monitor: Mutex>, + waiting_for_frame_until: Mutex>, +} + +#[derive(Clone, Copy)] +struct PendingMonitorRequest { + monitor_id: u32, + stalled_after_frame_seq: u64, + started_at: Instant, +} + +struct StoreFrameOutcome { + completed_ensure: bool, + completed_refresh: bool, } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamRequestProgress { + AwaitingFirstFrame, + Settled, +} + impl SharedLatestFrame { - fn store(&self, monitor_id: u32, frame: &QueuedPixelBufferFrame) { - match self.latest.lock() { + fn store(&self, monitor_id: u32, frame: &QueuedPixelBufferFrame) -> StoreFrameOutcome { + match self.frames.lock() { Ok(mut guard) => { - *guard = Some(LatestQueuedPixelBufferFrame { monitor_id, frame: frame.clone() }); + let shared = guard.get_or_insert_with(|| SharedQueuedPixelBufferFrames { + monitor_id, + frames: VecDeque::with_capacity(STREAM_FRAME_QUEUE_CAPACITY), + }); + if shared.monitor_id != monitor_id { + shared.monitor_id = monitor_id; + shared.frames.clear(); + } + if shared.frames.len() >= STREAM_FRAME_QUEUE_CAPACITY { + shared.frames.pop_front(); + } + shared.frames.push_back(frame.clone()); }, Err(poisoned) => { let mut guard = poisoned.into_inner(); - - *guard = Some(LatestQueuedPixelBufferFrame { monitor_id, frame: frame.clone() }); + let shared = guard.get_or_insert_with(|| SharedQueuedPixelBufferFrames { + monitor_id, + frames: VecDeque::with_capacity(STREAM_FRAME_QUEUE_CAPACITY), + }); + if shared.monitor_id != monitor_id { + shared.monitor_id = monitor_id; + shared.frames.clear(); + } + if shared.frames.len() >= STREAM_FRAME_QUEUE_CAPACITY { + shared.frames.pop_front(); + } + shared.frames.push_back(frame.clone()); }, } - self.finish_ensure_monitor(monitor_id); + self.complete_pending_requests_for_stored_frame(monitor_id) + } + + fn complete_pending_requests_for_stored_frame(&self, monitor_id: u32) -> StoreFrameOutcome { + self.clear_waiting_for_frame(monitor_id); + + StoreFrameOutcome { + completed_ensure: self.finish_ensure_monitor(monitor_id), + completed_refresh: self.finish_refresh_monitor(monitor_id), + } } fn latest_frame_for_monitor(&self, monitor_id: u32) -> Option { - match self.latest.lock() { + match self.frames.lock() { Ok(guard) => guard .as_ref() - .and_then(|latest| (latest.monitor_id == monitor_id).then(|| latest.frame.clone())), + .and_then(|latest| { + (latest.monitor_id == monitor_id).then(|| latest.frames.back().cloned()) + }) + .flatten(), Err(poisoned) => poisoned .into_inner() .as_ref() - .and_then(|latest| (latest.monitor_id == monitor_id).then(|| latest.frame.clone())), + .and_then(|latest| { + (latest.monitor_id == monitor_id).then(|| latest.frames.back().cloned()) + }) + .flatten(), + } + } + + fn frames_after_seq_for_monitor( + &self, + monitor_id: u32, + after_frame_seq: u64, + ) -> Vec { + match self.frames.lock() { + Ok(guard) => guard + .as_ref() + .filter(|shared| shared.monitor_id == monitor_id) + .map(|shared| { + shared + .frames + .iter() + .filter(|frame| frame.frame_seq > after_frame_seq) + .cloned() + .collect() + }) + .unwrap_or_default(), + Err(poisoned) => poisoned + .into_inner() + .as_ref() + .filter(|shared| shared.monitor_id == monitor_id) + .map(|shared| { + shared + .frames + .iter() + .filter(|frame| frame.frame_seq > after_frame_seq) + .cloned() + .collect() + }) + .unwrap_or_default(), } } @@ -478,11 +794,13 @@ impl SharedLatestFrame { true } - fn finish_ensure_monitor(&self, monitor_id: u32) { + fn finish_ensure_monitor(&self, monitor_id: u32) -> bool { match self.pending_monitor.lock() { Ok(mut guard) => { if guard.is_some_and(|pending_monitor_id| pending_monitor_id == monitor_id) { *guard = None; + + return true; } }, Err(poisoned) => { @@ -490,9 +808,209 @@ impl SharedLatestFrame { if guard.is_some_and(|pending_monitor_id| pending_monitor_id == monitor_id) { *guard = None; + + return true; + } + }, + } + + false + } + + fn begin_refresh_monitor( + &self, + monitor_id: u32, + stalled_after_frame_seq: u64, + now: Instant, + ) -> bool { + match self.pending_refresh_monitor.lock() { + Ok(mut guard) => { + if let Some(pending) = *guard { + if pending.monitor_id != monitor_id { + return false; + } + if pending.stalled_after_frame_seq != stalled_after_frame_seq { + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + + return true; + } + if now.saturating_duration_since(pending.started_at) + < STREAM_POST_SETUP_FRAME_GRACE + { + return false; + } + tracing::info!( + op = "live_frame_stream.stale_pending_refresh_recovered", + monitor_id, + stalled_after_frame_seq, + pending_age_ms = + now.saturating_duration_since(pending.started_at).as_millis() as u64, + "Recovered a stale pending ScreenCaptureKit refresh so a new refresh can be scheduled." + ); + + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + + return true; + } + + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + + if let Some(pending) = *guard { + if pending.monitor_id != monitor_id { + return false; + } + if pending.stalled_after_frame_seq != stalled_after_frame_seq { + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + + return true; + } + if now.saturating_duration_since(pending.started_at) + < STREAM_POST_SETUP_FRAME_GRACE + { + return false; + } + tracing::info!( + op = "live_frame_stream.stale_pending_refresh_recovered", + monitor_id, + stalled_after_frame_seq, + pending_age_ms = + now.saturating_duration_since(pending.started_at).as_millis() as u64, + "Recovered a stale pending ScreenCaptureKit refresh so a new refresh can be scheduled." + ); + + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + + return true; + } + + *guard = Some(PendingMonitorRequest { + monitor_id, + stalled_after_frame_seq, + started_at: now, + }); + }, + } + + true + } + + fn mark_waiting_for_frame(&self, monitor_id: u32) { + self.mark_waiting_for_frame_until( + monitor_id, + Instant::now() + STREAM_POST_SETUP_FRAME_GRACE, + ); + } + + fn mark_waiting_for_frame_until(&self, monitor_id: u32, until: Instant) { + match self.waiting_for_frame_until.lock() { + Ok(mut guard) => { + *guard = Some((monitor_id, until)); + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + + *guard = Some((monitor_id, until)); + }, + } + } + + fn waiting_for_frame_after_setup(&self, monitor_id: u32) -> bool { + self.waiting_for_frame_after_setup_at(monitor_id, Instant::now()) + } + + fn waiting_for_frame_after_setup_at(&self, monitor_id: u32, now: Instant) -> bool { + match self.waiting_for_frame_until.lock() { + Ok(mut guard) => { + let Some((pending_monitor_id, until)) = *guard else { + return false; + }; + if pending_monitor_id != monitor_id { + return false; + } + if now < until { + return true; + } + *guard = None; + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + let Some((pending_monitor_id, until)) = *guard else { + return false; + }; + if pending_monitor_id != monitor_id { + return false; + } + if now < until { + return true; + } + *guard = None; + }, + } + + false + } + + fn clear_waiting_for_frame(&self, monitor_id: u32) { + match self.waiting_for_frame_until.lock() { + Ok(mut guard) => { + if guard.is_some_and(|(pending_monitor_id, _)| pending_monitor_id == monitor_id) { + *guard = None; + } + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + + if guard.is_some_and(|(pending_monitor_id, _)| pending_monitor_id == monitor_id) { + *guard = None; + } + }, + } + } + + fn finish_refresh_monitor(&self, monitor_id: u32) -> bool { + match self.pending_refresh_monitor.lock() { + Ok(mut guard) => { + if guard.is_some_and(|pending| pending.monitor_id == monitor_id) { + *guard = None; + + return true; + } + }, + Err(poisoned) => { + let mut guard = poisoned.into_inner(); + + if guard.is_some_and(|pending| pending.monitor_id == monitor_id) { + *guard = None; + + return true; } }, } + + false } } @@ -525,6 +1043,7 @@ struct StreamState { self_capture_exception_window_ids_complete: bool, stream: Retained, output: Retained, + sample_handler_queue: DispatchRetained, } struct CurrentProcessExceptionWindows { @@ -593,125 +1112,433 @@ fn stream_worker_loop( frame_waker: Option>, shared_latest_frame: Arc, filter: StreamFilterConfig, + capture_target: StreamCaptureTarget, ) { let frame_seq_counter = Arc::new(AtomicU64::new(0)); let mut state: Option = None; let mut last_setup_attempt_at: Option = None; while let Ok(request) = request_rx.recv() { - match request { - WorkerRequest::EnsureMonitor { monitor } => { - let _ = ensure_stream( - &mut state, - &mut last_setup_attempt_at, - STREAM_SETUP_BACKOFF, - monitor, - &filter, - frame_waker.clone(), - frame_seq_counter.clone(), - shared_latest_frame.clone(), - ); + if !handle_stream_worker_request( + request, + &mut state, + &mut last_setup_attempt_at, + &filter, + capture_target, + frame_waker.clone(), + frame_seq_counter.clone(), + shared_latest_frame.clone(), + ) { + break; + } + } - shared_latest_frame.finish_ensure_monitor(monitor.id); - }, - WorkerRequest::SampleCursor { + teardown_stream(&mut state); +} + +#[allow(clippy::too_many_arguments)] +fn handle_stream_worker_request( + request: WorkerRequest, + state: &mut Option, + last_setup_attempt_at: &mut Option, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +) -> bool { + match request { + WorkerRequest::EnsureMonitor { monitor } => handle_ensure_monitor_request( + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ), + WorkerRequest::RefreshMonitor { monitor } => handle_refresh_monitor_request( + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ), + WorkerRequest::SampleCursor { + monitor, + x_px, + y_px, + want_patch, + patch_width_px, + patch_height_px, + reply_tx, + } => { + reply_with_sample_cursor( + state, + last_setup_attempt_at, monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, x_px, y_px, want_patch, patch_width_px, patch_height_px, reply_tx, - } => { - let rgb = ensure_stream( - &mut state, - &mut last_setup_attempt_at, - STREAM_SETUP_BACKOFF, - monitor, - &filter, - frame_waker.clone(), - frame_seq_counter.clone(), - shared_latest_frame.clone(), - ) - .and_then(|_| { - let stream_state = state.as_ref()?; - - stream_state.output.latest_pixel_buffer().and_then(|pixel_buffer| { - sample_cursor_from_pixel_buffer( - &pixel_buffer, - x_px, - y_px, - want_patch, - patch_width_px, - patch_height_px, - ) - }) - }); - let _ = reply_tx.send(rgb); - }, - WorkerRequest::LatestRgbaSnapshot { monitor, reply_tx } => { - let snapshot = ensure_stream( - &mut state, - &mut last_setup_attempt_at, - STREAM_SETUP_BACKOFF, - monitor, - &filter, - frame_waker.clone(), - frame_seq_counter.clone(), - shared_latest_frame.clone(), - ) - .and_then(|_| { - let stream_state = state.as_ref()?; - let frame = stream_state.output.latest_frame()?; - let (width_px, height_px) = pixel_buffer_size_px(&frame.pixel_buffer)?; - let image = - rgba_image_from_pixel_buffer(&frame.pixel_buffer, width_px, height_px)?; - - Some(Arc::new(MonitorImageSnapshot { - captured_at: frame.captured_at, - monitor, - image: Arc::new(image), - })) - }); - let _ = reply_tx.send(snapshot); - }, - WorkerRequest::LatestRgbaRegion { monitor, rect_px, reply_tx } => { - let image = latest_fresh_rgba_region( - &mut state, - &mut last_setup_attempt_at, - monitor, - rect_px, - &filter, - frame_waker.clone(), - frame_seq_counter.clone(), - shared_latest_frame.clone(), - ); - let _ = reply_tx.send(image); - }, - WorkerRequest::OrderedRgbaRegionsAfterSeq { + ); + true + }, + WorkerRequest::LatestRgbaSnapshot { monitor, reply_tx } => { + reply_with_latest_rgba_snapshot( + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + reply_tx, + ); + true + }, + WorkerRequest::LatestRgbaRegion { monitor, rect_px, reply_tx } => { + reply_with_latest_rgba_region( + state, + last_setup_attempt_at, + monitor, + rect_px, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + reply_tx, + ); + true + }, + WorkerRequest::OrderedRgbaRegionsAfterSeq { + monitor, + rect_px, + after_frame_seq, + reply_tx, + } => { + reply_with_ordered_rgba_regions_after_seq( + state, + last_setup_attempt_at, monitor, rect_px, after_frame_seq, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, reply_tx, - } => { - let frames = ordered_fresh_rgba_regions_after_seq( - &mut state, - &mut last_setup_attempt_at, - monitor, - rect_px, - after_frame_seq, - &filter, - frame_waker.clone(), - frame_seq_counter.clone(), - shared_latest_frame.clone(), - ); - let _ = reply_tx.send(frames); - }, - WorkerRequest::Shutdown => break, - } + false, + ); + true + }, + WorkerRequest::Shutdown => false, + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_ensure_monitor_request( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +) -> bool { + tracing::info!( + op = "live_frame_stream.ensure_monitor_begin", + monitor_id = monitor.id, + current_monitor_id = state.as_ref().map(|current| current.monitor_id), + "Handling an asynchronous ScreenCaptureKit ensure request." + ); + let progress = ensure_stream( + state, + last_setup_attempt_at, + STREAM_SETUP_BACKOFF, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame.clone(), + ); + if progress == StreamRequestProgress::Settled { + shared_latest_frame.finish_ensure_monitor(monitor.id); + } + + true +} + +#[allow(clippy::too_many_arguments)] +fn handle_refresh_monitor_request( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +) -> bool { + if shared_latest_frame.waiting_for_frame_after_setup(monitor.id) { + tracing::info!( + op = "live_frame_stream.refresh_monitor_skipped_waiting_for_first_frame", + monitor_id = monitor.id, + current_monitor_id = state.as_ref().map(|current| current.monitor_id), + pending_refresh_preserved = true, + "Skipped a queued ScreenCaptureKit refresh because the stream is still waiting for the first frame from the previous setup." + ); + + return true; + } + + tracing::info!( + op = "live_frame_stream.refresh_monitor_begin", + monitor_id = monitor.id, + current_monitor_id = state.as_ref().map(|current| current.monitor_id), + "Handling an asynchronous ScreenCaptureKit refresh request." + ); + let progress = refresh_stream_nonblocking( + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame.clone(), + ); + if progress == StreamRequestProgress::Settled { + shared_latest_frame.finish_refresh_monitor(monitor.id); + } + + true +} + +#[allow(clippy::too_many_arguments)] +fn reply_with_sample_cursor( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, + x_px: u32, + y_px: u32, + want_patch: bool, + patch_width_px: u32, + patch_height_px: u32, + reply_tx: Sender>, +) { + let _ = ensure_stream( + state, + last_setup_attempt_at, + STREAM_SETUP_BACKOFF, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ); + let rgb = state.as_ref().and_then(|stream_state| { + stream_state.output.latest_pixel_buffer().and_then(|pixel_buffer| { + sample_cursor_from_pixel_buffer( + &pixel_buffer, + x_px, + y_px, + want_patch, + patch_width_px, + patch_height_px, + ) + }) + }); + let _ = reply_tx.send(rgb); +} + +#[allow(clippy::too_many_arguments)] +fn reply_with_latest_rgba_snapshot( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, + reply_tx: Sender>>, +) { + let _ = ensure_stream( + state, + last_setup_attempt_at, + STREAM_SETUP_BACKOFF, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ); + let snapshot = state.as_ref().and_then(|stream_state| { + let frame = stream_state.output.latest_frame()?; + let (width_px, height_px) = pixel_buffer_size_px(&frame.pixel_buffer)?; + let image = rgba_image_from_pixel_buffer(&frame.pixel_buffer, width_px, height_px)?; + + Some(Arc::new(MonitorImageSnapshot { + captured_at: frame.captured_at, + monitor, + image: Arc::new(image), + })) + }); + let _ = reply_tx.send(snapshot); +} + +#[allow(clippy::too_many_arguments)] +fn reply_with_latest_rgba_region( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + rect_px: RectPoints, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, + reply_tx: Sender>, +) { + let image = latest_fresh_rgba_region( + state, + last_setup_attempt_at, + monitor, + rect_px, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ); + let _ = reply_tx.send(image); +} + +#[allow(clippy::too_many_arguments)] +fn reply_with_ordered_rgba_regions_after_seq( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + rect_px: RectPoints, + after_frame_seq: u64, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, + reply_tx: Sender>>, + nonblocking: bool, +) { + let frames = if nonblocking { + ordered_queued_rgba_regions_after_seq_nonblocking( + state, + last_setup_attempt_at, + monitor, + rect_px, + after_frame_seq, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ) + } else { + ordered_fresh_rgba_regions_after_seq( + state, + last_setup_attempt_at, + monitor, + rect_px, + after_frame_seq, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ) + }; + let _ = reply_tx.send(frames); +} + +#[allow(clippy::too_many_arguments)] +fn refresh_stream_nonblocking( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +) -> StreamRequestProgress { + let now = Instant::now(); + + if let Some(last) = *last_setup_attempt_at + && now.duration_since(last) < STREAM_SETUP_BACKOFF + { + tracing::info!( + op = "live_frame_stream.refresh_monitor_backoff", + monitor_id = monitor.id, + elapsed_since_last_setup_ms = now.duration_since(last).as_millis(), + backoff_ms = STREAM_SETUP_BACKOFF.as_millis(), + "Skipped ScreenCaptureKit refresh because setup backoff is still active." + ); + return StreamRequestProgress::Settled; } - teardown_stream(&mut state); + if state.as_ref().is_none_or(|current| current.monitor_id != monitor.id) { + tracing::info!( + op = "live_frame_stream.refresh_monitor_recover_via_ensure", + monitor_id = monitor.id, + current_monitor_id = state.as_ref().map(|current| current.monitor_id), + "Refresh request found no matching live stream and is falling back to ensure." + ); + return ensure_stream( + state, + last_setup_attempt_at, + STREAM_SETUP_BACKOFF, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ); + } + + refresh_stream(RefreshStreamArgs { + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + }) } #[allow(clippy::too_many_arguments)] @@ -721,10 +1548,11 @@ fn ensure_stream( setup_backoff: Duration, monitor: MonitorRect, filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, frame_waker: Option>, frame_seq_counter: Arc, shared_latest_frame: Arc, -) -> Option<()> { +) -> StreamRequestProgress { let reuse_decision = stream_reuse_decision( state.as_ref().map(|current| current.monitor_id), state.as_ref().is_some_and(|current| current.self_capture_exception_window_ids_complete), @@ -733,13 +1561,23 @@ fn ensure_stream( let setup_backoff = stream_setup_backoff(reuse_decision, setup_backoff); if reuse_decision == StreamReuseDecision::ReuseCurrent { - return Some(()); + return StreamRequestProgress::Settled; } let now = Instant::now(); - if last_setup_attempt_at.is_some_and(|t| now.duration_since(t) < setup_backoff) { - return (reuse_decision == StreamReuseDecision::RetryUpgradeUsingCurrent).then_some(()); + if let Some(last_attempt_at) = *last_setup_attempt_at + && now.duration_since(last_attempt_at) < setup_backoff + { + tracing::info!( + op = "live_frame_stream.ensure_stream_backoff", + monitor_id = monitor.id, + reuse_decision = ?reuse_decision, + elapsed_since_last_setup_ms = now.duration_since(last_attempt_at).as_millis(), + backoff_ms = setup_backoff.as_millis(), + "Skipped ScreenCaptureKit setup because the current setup backoff window is still active." + ); + return StreamRequestProgress::Settled; } *last_setup_attempt_at = Some(now); @@ -747,34 +1585,66 @@ fn ensure_stream( let Some(next_state) = setup_stream_for_monitor( monitor, filter, + capture_target, frame_waker, frame_seq_counter, - shared_latest_frame, + shared_latest_frame.clone(), ) else { - return (reuse_decision == StreamReuseDecision::RetryUpgradeUsingCurrent).then_some(()); + tracing::warn!( + op = "live_frame_stream.ensure_stream_setup_failed", + monitor_id = monitor.id, + reuse_decision = ?reuse_decision, + had_existing_state = state.is_some(), + "ScreenCaptureKit setup did not produce a usable live stream." + ); + return StreamRequestProgress::Settled; }; if reuse_decision == StreamReuseDecision::RetryUpgradeUsingCurrent { if !next_state.self_capture_exception_window_ids_complete { + tracing::info!( + op = "live_frame_stream.ensure_stream_upgrade_deferred", + monitor_id = monitor.id, + "Retained the current live stream because the replacement setup still lacked complete self-capture exception windows." + ); let mut next_state = Some(next_state); teardown_stream(&mut next_state); - return Some(()); + return StreamRequestProgress::Settled; } let mut previous_state = state.replace(next_state); teardown_stream(&mut previous_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( + op = "live_frame_stream.ensure_stream_ready", + monitor_id = monitor.id, + reuse_decision = ?reuse_decision, + self_capture_exception_window_ids_complete = true, + replaced_existing_state = true, + "ScreenCaptureKit setup replaced the existing live stream." + ); - return Some(()); + return StreamRequestProgress::AwaitingFirstFrame; } teardown_stream(state); + let exceptions_complete = next_state.self_capture_exception_window_ids_complete; *state = Some(next_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( + op = "live_frame_stream.ensure_stream_ready", + monitor_id = monitor.id, + reuse_decision = ?reuse_decision, + self_capture_exception_window_ids_complete = exceptions_complete, + replaced_existing_state = false, + "ScreenCaptureKit setup produced a live stream." + ); - Some(()) + StreamRequestProgress::AwaitingFirstFrame } #[allow(clippy::too_many_arguments)] @@ -784,20 +1654,23 @@ fn latest_fresh_rgba_region( monitor: MonitorRect, rect_px: RectPoints, filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, frame_waker: Option>, frame_seq_counter: Arc, shared_latest_frame: Arc, ) -> Option { - ensure_stream( + let stream_rect_px = stream_rect_for_requested_region(capture_target, rect_px)?; + let _ = ensure_stream( state, last_setup_attempt_at, STREAM_SETUP_BACKOFF, monitor, filter, + capture_target, frame_waker.clone(), frame_seq_counter.clone(), shared_latest_frame.clone(), - )?; + ); let now = Instant::now(); let stream_state = state.as_ref()?; @@ -805,18 +1678,19 @@ fn latest_fresh_rgba_region( if let Some(frame) = stream_state.output.latest_frame() && now.saturating_duration_since(frame.captured_at) <= STREAM_REGION_FRAME_MAX_AGE { - return rgba_region_from_pixel_buffer(&frame.pixel_buffer, rect_px); + return rgba_region_from_pixel_buffer(&frame.pixel_buffer, stream_rect_px); } - refresh_stream( + let _ = refresh_stream(RefreshStreamArgs { state, last_setup_attempt_at, monitor, filter, + capture_target, frame_waker, frame_seq_counter, shared_latest_frame, - )?; + }); let min_captured_at = Instant::now(); let deadline = min_captured_at + STREAM_REGION_FRAME_REFRESH_TIMEOUT; @@ -827,7 +1701,7 @@ fn latest_fresh_rgba_region( if let Some(frame) = stream_state.output.latest_frame() && frame.captured_at >= min_captured_at { - return rgba_region_from_pixel_buffer(&frame.pixel_buffer, rect_px); + return rgba_region_from_pixel_buffer(&frame.pixel_buffer, stream_rect_px); } if Instant::now() >= deadline { @@ -838,6 +1712,39 @@ fn latest_fresh_rgba_region( } } +#[allow(clippy::too_many_arguments)] +fn ordered_queued_rgba_regions_after_seq_nonblocking( + state: &mut Option, + last_setup_attempt_at: &mut Option, + monitor: MonitorRect, + rect_px: RectPoints, + after_frame_seq: u64, + filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +) -> Option> { + let stream_rect_px = stream_rect_for_requested_region(capture_target, rect_px)?; + let _ = ensure_stream( + state, + last_setup_attempt_at, + STREAM_SETUP_BACKOFF, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + ); + + let stream_state = state.as_ref()?; + let frames = stream_state.output.queued_frames_after_seq(after_frame_seq); + let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); + + (!frames.is_empty()).then_some(frames) +} + #[allow(clippy::too_many_arguments)] fn ordered_fresh_rgba_regions_after_seq( state: &mut Option, @@ -846,24 +1753,27 @@ fn ordered_fresh_rgba_regions_after_seq( rect_px: RectPoints, after_frame_seq: u64, filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, frame_waker: Option>, frame_seq_counter: Arc, shared_latest_frame: Arc, ) -> Option> { - ensure_stream( + let stream_rect_px = stream_rect_for_requested_region(capture_target, rect_px)?; + let _ = ensure_stream( state, last_setup_attempt_at, STREAM_SETUP_BACKOFF, monitor, filter, + capture_target, frame_waker.clone(), frame_seq_counter.clone(), shared_latest_frame.clone(), - )?; + ); let stream_state = state.as_ref()?; let frames = stream_state.output.queued_frames_after_seq(after_frame_seq); - let frames = ordered_rgba_regions_from_frames(frames, rect_px); + let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); if !frames.is_empty() { return Some(frames); @@ -877,15 +1787,16 @@ fn ordered_fresh_rgba_regions_after_seq( return None; } - refresh_stream( + let _ = refresh_stream(RefreshStreamArgs { state, last_setup_attempt_at, monitor, filter, + capture_target, frame_waker, frame_seq_counter, shared_latest_frame, - )?; + }); let min_captured_at = Instant::now(); let deadline = min_captured_at + STREAM_REGION_FRAME_REFRESH_TIMEOUT; @@ -893,7 +1804,7 @@ fn ordered_fresh_rgba_regions_after_seq( loop { let stream_state = state.as_ref()?; let frames = stream_state.output.queued_frames_after_seq(after_frame_seq); - let frames = ordered_rgba_regions_from_frames(frames, rect_px); + let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); if !frames.is_empty() { return Some(frames); @@ -906,34 +1817,72 @@ fn ordered_fresh_rgba_regions_after_seq( } } -fn refresh_stream( - state: &mut Option, - last_setup_attempt_at: &mut Option, - monitor: MonitorRect, - filter: &StreamFilterConfig, - frame_waker: Option>, - frame_seq_counter: Arc, - shared_latest_frame: Arc, -) -> Option<()> { +fn refresh_stream(args: RefreshStreamArgs<'_>) -> StreamRequestProgress { + let RefreshStreamArgs { + state, + last_setup_attempt_at, + monitor, + filter, + capture_target, + frame_waker, + frame_seq_counter, + shared_latest_frame, + } = args; + tracing::info!( + op = "live_frame_stream.refresh_stream_begin", + monitor_id = monitor.id, + current_monitor_id = state.as_ref().map(|current| current.monitor_id), + "Refreshing the ScreenCaptureKit live stream." + ); *last_setup_attempt_at = Some(Instant::now()); - teardown_stream(state); - - *state = Some(setup_stream_for_monitor( + let Some(next_state) = setup_stream_for_monitor( monitor, filter, + capture_target, frame_waker, frame_seq_counter, - shared_latest_frame, - )?); + shared_latest_frame.clone(), + ) else { + return StreamRequestProgress::Settled; + }; + let exceptions_complete = next_state.self_capture_exception_window_ids_complete; + let replaced_existing_state = state.is_some(); + let mut previous_state = state.replace(next_state); + + teardown_stream(&mut previous_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( + op = "live_frame_stream.refresh_stream_ready", + monitor_id = monitor.id, + self_capture_exception_window_ids_complete = exceptions_complete, + replaced_existing_state, + "Refresh completed and installed a new ScreenCaptureKit live stream." + ); - Some(()) + StreamRequestProgress::AwaitingFirstFrame +} + +struct RefreshStreamArgs<'a> { + state: &'a mut Option, + last_setup_attempt_at: &'a mut Option, + monitor: MonitorRect, + filter: &'a StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, } fn teardown_stream(state: &mut Option) { let Some(state) = state.take() else { return; }; + tracing::info!( + op = "live_frame_stream.teardown_stream", + monitor_id = state.monitor_id, + "Stopping the current ScreenCaptureKit live stream." + ); let stop_block = RcBlock::new(|_err: *mut NSError| {}); unsafe { state.stream.stopCaptureWithCompletionHandler(Some(&stop_block)) }; @@ -942,12 +1891,13 @@ fn teardown_stream(state: &mut Option) { fn setup_stream_for_monitor( monitor: MonitorRect, filter: &StreamFilterConfig, + capture_target: StreamCaptureTarget, frame_waker: Option>, frame_seq_counter: Arc, shared_latest_frame: Arc, ) -> Option { - let content = get_shareable_content().ok()?; - let display = find_display(&content, monitor.id)?; + let content = load_shareable_content_for_monitor(monitor.id)?; + let display = find_display_for_monitor(&content, monitor.id)?; let excepting_windows = find_current_process_exception_windows(&content, &filter.self_capture_exception_window_ids); let filter_mode = stream_filter_mode_for_current_process(excepting_windows.complete()); @@ -1009,7 +1959,8 @@ fn setup_stream_for_monitor( } }, }; - let config = build_stream_config_for_monitor(monitor); + let config = build_stream_config_for_monitor(monitor, capture_target); + let sample_handler_queue = build_sample_handler_queue_for_monitor(monitor.id); let output = StreamOutput::new(monitor.id, frame_waker, frame_seq_counter, shared_latest_frame); let delegate_proto = ProtocolObject::from_ref(&*output); let stream = unsafe { @@ -1026,25 +1977,94 @@ fn setup_stream_for_monitor( stream.addStreamOutput_type_sampleHandlerQueue_error( output_proto, SCStreamOutputType::Screen, - None, + Some(&sample_handler_queue), ) } .is_err() { + log_add_stream_output_failed(monitor.id, filter_mode); return None; } - if start_capture_blocking(&stream).is_err() { + if let Err(error) = start_capture_blocking(&stream) { + log_start_capture_failed(monitor.id, filter_mode, &error); return None; } + tracing::info!( + op = "live_frame_stream.setup_stream_ready", + monitor_id = monitor.id, + filter_mode = ?filter_mode, + self_capture_exception_window_ids_complete = excepting_windows.complete(), + excepting_window_count = excepting_windows.windows.len(), + fallback_excluded_window_count = excepting_windows.fallback_excluded_windows.len(), + missing_window_ids = ?excepting_windows.missing_window_ids, + "ScreenCaptureKit setup created a live stream for the requested monitor." + ); Some(StreamState { monitor_id: monitor.id, self_capture_exception_window_ids_complete: excepting_windows.complete(), stream, output, + sample_handler_queue, }) } +fn load_shareable_content_for_monitor(monitor_id: u32) -> Option> { + match get_shareable_content() { + Ok(content) => Some(content), + Err(error) => { + tracing::warn!( + op = "live_frame_stream.get_shareable_content_failed", + monitor_id, + error_code = error.code(), + error_domain = %error.domain(), + error_description = %error.localizedDescription(), + "Failed to load ScreenCaptureKit shareable content during live stream setup." + ); + + None + }, + } +} + +fn find_display_for_monitor( + content: &SCShareableContent, + monitor_id: u32, +) -> Option> { + let Some(display) = find_display(content, monitor_id) else { + tracing::warn!( + op = "live_frame_stream.find_display_failed", + monitor_id, + "Failed to find the requested monitor in ScreenCaptureKit shareable content." + ); + + return None; + }; + + Some(display) +} + +fn log_add_stream_output_failed(monitor_id: u32, filter_mode: StreamFilterMode) { + tracing::warn!( + op = "live_frame_stream.add_stream_output_failed", + monitor_id, + filter_mode = ?filter_mode, + "Failed to register the ScreenCaptureKit stream output." + ); +} + +fn log_start_capture_failed(monitor_id: u32, filter_mode: StreamFilterMode, error: &NSError) { + tracing::warn!( + op = "live_frame_stream.start_capture_failed", + monitor_id, + filter_mode = ?filter_mode, + error_code = error.code(), + error_domain = %error.domain(), + error_description = %error.localizedDescription(), + "ScreenCaptureKit failed to start the live stream." + ); +} + fn find_current_process_exception_windows( content: &SCShareableContent, self_capture_exception_window_ids: &[u32], @@ -1262,12 +2282,21 @@ fn find_display(content: &SCShareableContent, monitor_id: u32) -> Option Retained { +fn build_stream_config_for_monitor( + monitor: MonitorRect, + capture_target: StreamCaptureTarget, +) -> Retained { let config = unsafe { SCStreamConfiguration::new() }; - // Prefer full-resolution capture. let sf = monitor.scale_factor().max(1.0); - let width_px = ((monitor.width as f32) * sf).round().max(1.0) as usize; - let height_px = ((monitor.height as f32) * sf).round().max(1.0) as usize; + let (width_px, height_px) = match capture_target { + StreamCaptureTarget::FullMonitor => ( + ((monitor.width as f32) * sf).round().max(1.0) as usize, + ((monitor.height as f32) * sf).round().max(1.0) as usize, + ), + StreamCaptureTarget::Region(region) => { + (region.rect_pixels.width.max(1) as usize, region.rect_pixels.height.max(1) as usize) + }, + }; unsafe { config.setWidth(width_px) }; unsafe { config.setHeight(height_px) }; @@ -1280,12 +2309,30 @@ fn build_stream_config_for_monitor(monitor: MonitorRect) -> Retained DispatchRetained { + DispatchQueue::new(&sample_handler_queue_label(monitor_id), DispatchQueueAttr::SERIAL) +} + +fn sample_handler_queue_label(monitor_id: u32) -> String { + format!("io.hackink.rsnap.scroll-capture.sample-handler.monitor-{monitor_id}") +} + fn pixel_buffer_size_px(pixel_buffer: &CFRetained) -> Option<(u32, u32)> { let width = objc2_core_video::CVPixelBufferGetWidth(pixel_buffer); let height = objc2_core_video::CVPixelBufferGetHeight(pixel_buffer); @@ -1516,11 +2563,35 @@ fn ordered_rgba_regions_from_frames( #[cfg(test)] mod tests { + use std::sync::{Arc, atomic::AtomicU64}; use std::time::Duration; + use std::{ptr, ptr::NonNull}; - use crate::live_frame_stream_macos::{self, StreamFilterMode}; + use objc2_core_foundation::CFRetained; + use objc2_core_video::{CVPixelBufferCreate, kCVPixelFormatType_32BGRA, kCVReturnSuccess}; + + use crate::live_frame_stream_macos::{self, STREAM_POST_SETUP_FRAME_GRACE, StreamFilterMode}; use crate::state::Rgb; + fn test_pixel_buffer() -> live_frame_stream_macos::SharedPixelBuffer { + let mut buffer = ptr::null_mut(); + let res = unsafe { + CVPixelBufferCreate( + None, + 1, + 1, + kCVPixelFormatType_32BGRA, + None, + NonNull::from(&mut buffer), + ) + }; + assert_eq!(res, kCVReturnSuccess); + + live_frame_stream_macos::SharedPixelBuffer(unsafe { + CFRetained::from_raw(NonNull::new(buffer).expect("test pixel buffer")) + }) + } + #[test] fn stream_filter_mode_prefers_process_exclusion_only_when_exception_list_is_complete() { assert_eq!( @@ -1590,6 +2661,312 @@ mod tests { ); } + #[test] + fn waiting_for_first_frame_expires_after_grace_window() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + let until = now + Duration::from_millis(50); + + shared.mark_waiting_for_frame_until(7, until); + + assert!(shared.waiting_for_frame_after_setup_at(7, now + Duration::from_millis(25))); + assert!(!shared.waiting_for_frame_after_setup_at(7, now + Duration::from_millis(60))); + assert!(!shared.waiting_for_frame_after_setup_at(7, now + Duration::from_millis(61))); + } + + #[test] + fn stored_frame_completion_clears_pending_ensure_for_same_monitor() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + + assert!(shared.begin_ensure_monitor(7)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); + + let outcome = shared.complete_pending_requests_for_stored_frame(7); + + assert!(outcome.completed_ensure); + assert!(!outcome.completed_refresh); + assert!(!shared.waiting_for_frame_after_setup_at(7, now)); + assert!(!shared.finish_ensure_monitor(7)); + } + + #[test] + fn stored_frame_completion_leaves_other_monitor_refresh_pending() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + + assert!(shared.begin_refresh_monitor(7, 11, now)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); + + let outcome = shared.complete_pending_requests_for_stored_frame(9); + + assert!(!outcome.completed_ensure); + assert!(!outcome.completed_refresh); + assert!(shared.waiting_for_frame_after_setup_at(7, now)); + assert!(shared.finish_refresh_monitor(7)); + } + + #[test] + fn stored_frame_completion_clears_pending_refresh_for_same_monitor() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + + assert!(shared.begin_refresh_monitor(7, 11, now)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); + + let outcome = shared.complete_pending_requests_for_stored_frame(7); + + assert!(!outcome.completed_ensure); + assert!(outcome.completed_refresh); + assert!(!shared.waiting_for_frame_after_setup_at(7, now)); + assert!(!shared.finish_refresh_monitor(7)); + } + + #[test] + fn stale_pending_refresh_retries_again_after_each_grace_window_for_same_stalled_frontier() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + + assert!(shared.begin_refresh_monitor(7, 11, now)); + assert!(!shared.begin_refresh_monitor(7, 11, now + Duration::from_millis(100))); + assert!(shared.begin_refresh_monitor( + 7, + 11, + now + STREAM_POST_SETUP_FRAME_GRACE + Duration::from_millis(1), + )); + assert!(!shared.begin_refresh_monitor( + 7, + 11, + now + STREAM_POST_SETUP_FRAME_GRACE + Duration::from_millis(2), + )); + assert!(shared.begin_refresh_monitor( + 7, + 11, + now + STREAM_POST_SETUP_FRAME_GRACE.saturating_mul(2) + Duration::from_millis(1), + )); + } + + #[test] + fn stale_pending_refresh_rearms_when_stalled_frontier_advances() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let now = std::time::Instant::now(); + + assert!(shared.begin_refresh_monitor(7, 11, now)); + assert!(shared.begin_refresh_monitor( + 7, + 11, + now + STREAM_POST_SETUP_FRAME_GRACE + Duration::from_millis(1), + )); + assert!(shared.begin_refresh_monitor( + 7, + 12, + now + STREAM_POST_SETUP_FRAME_GRACE + Duration::from_millis(2), + )); + } + + #[test] + fn queued_refresh_request_stays_pending_while_waiting_for_previous_first_frame() { + let shared = Arc::new(live_frame_stream_macos::SharedLatestFrame::default()); + let now = std::time::Instant::now(); + let monitor = crate::state::MonitorRect { + id: 7, + origin: crate::state::GlobalPoint::new(0, 0), + width: 1440, + height: 900, + scale_factor_x1000: 2_000, + }; + let mut state = None; + let mut last_setup_attempt_at = None; + + assert!(shared.begin_refresh_monitor(monitor.id, 11, now)); + shared.mark_waiting_for_frame_until(monitor.id, now + Duration::from_secs(1)); + + assert!(live_frame_stream_macos::handle_refresh_monitor_request( + &mut state, + &mut last_setup_attempt_at, + monitor, + &live_frame_stream_macos::StreamFilterConfig::default(), + live_frame_stream_macos::StreamCaptureTarget::FullMonitor, + None, + Arc::new(AtomicU64::new(0)), + shared.clone(), + )); + assert!(state.is_none()); + assert!(shared.finish_refresh_monitor(monitor.id)); + } + + #[test] + fn shared_frame_history_returns_all_frames_after_frontier() { + let shared = live_frame_stream_macos::SharedLatestFrame::default(); + let monitor_id = 7; + let pixel_buffer = test_pixel_buffer(); + let make_frame = |frame_seq| live_frame_stream_macos::QueuedPixelBufferFrame { + frame_seq, + captured_at: std::time::Instant::now(), + pixel_buffer: pixel_buffer.clone(), + }; + + for frame_seq in 1..=4 { + let frame = make_frame(frame_seq); + let _ = shared.store(monitor_id, &frame); + } + + let queued = shared.frames_after_seq_for_monitor(monitor_id, 1); + let seqs: Vec = queued.into_iter().map(|frame| frame.frame_seq).collect(); + + assert_eq!(seqs, vec![2, 3, 4]); + } + + #[test] + fn nonblocking_after_seq_query_does_not_prime_when_same_monitor_already_has_latest_frame() { + let mut stream = live_frame_stream_macos::MacLiveFrameStream::with_waker(None); + let monitor = crate::state::MonitorRect { + id: 7, + origin: crate::state::GlobalPoint::new(0, 0), + width: 1440, + height: 900, + scale_factor_x1000: 2_000, + }; + let rect = crate::state::RectPoints::new(0, 0, 1, 1); + let frame = live_frame_stream_macos::QueuedPixelBufferFrame { + frame_seq: 4, + captured_at: std::time::Instant::now(), + pixel_buffer: test_pixel_buffer(), + }; + + let _ = stream.shared_latest_frame.store(monitor.id, &frame); + + assert!(stream.ordered_rgba_regions_after_seq_nonblocking(monitor, rect, 4).is_none()); + assert!( + stream + .shared_latest_frame + .pending_monitor + .lock() + .expect("pending_monitor lock") + .is_none() + ); + } + + #[test] + fn force_refresh_immediately_refreshes_when_seq_is_stalled() { + assert!(live_frame_stream_macos::should_refresh_monitor_frame( + 7, + 7, + Duration::from_millis(0), + true, + )); + assert!(!live_frame_stream_macos::should_refresh_monitor_frame( + 7, + 7, + Duration::from_millis(10), + false, + )); + } + + #[test] + fn force_refresh_does_not_refresh_when_newer_frame_already_exists() { + assert!(!live_frame_stream_macos::should_refresh_monitor_frame( + 8, + 7, + Duration::from_millis(200), + true, + )); + } + + #[test] + fn stream_config_uses_cadence_queue_depth_contract() { + let monitor = crate::state::MonitorRect { + id: 7, + origin: crate::state::GlobalPoint::new(0, 0), + width: 1440, + height: 900, + scale_factor_x1000: 2_000, + }; + let config = live_frame_stream_macos::build_stream_config_for_monitor( + monitor, + live_frame_stream_macos::StreamCaptureTarget::FullMonitor, + ); + + assert_eq!( + unsafe { config.queueDepth() }, + live_frame_stream_macos::STREAM_CONFIG_QUEUE_DEPTH as isize + ); + } + + #[test] + fn stream_config_uses_source_rect_for_scroll_capture_region() { + let monitor = crate::state::MonitorRect { + id: 7, + origin: crate::state::GlobalPoint::new(0, 0), + width: 1440, + height: 900, + scale_factor_x1000: 2_000, + }; + let region_points = crate::state::RectPoints::new(120, 80, 300, 220); + let region_pixels = monitor.local_rect_to_pixels(region_points); + let config = live_frame_stream_macos::build_stream_config_for_monitor( + monitor, + live_frame_stream_macos::StreamCaptureTarget::Region( + live_frame_stream_macos::StreamCaptureRegion { + rect_points: region_points, + rect_pixels: region_pixels, + }, + ), + ); + let source_rect = unsafe { config.sourceRect() }; + + assert_eq!(unsafe { config.width() }, region_pixels.width as usize); + assert_eq!(unsafe { config.height() }, region_pixels.height as usize); + assert_eq!(source_rect.origin.x, f64::from(region_points.x)); + assert_eq!(source_rect.origin.y, f64::from(region_points.y)); + assert_eq!(source_rect.size.width, f64::from(region_points.width)); + assert_eq!(source_rect.size.height, f64::from(region_points.height)); + } + + #[test] + fn stream_rect_maps_scroll_capture_region_requests_to_stream_local_rect() { + let capture_target = live_frame_stream_macos::StreamCaptureTarget::Region( + live_frame_stream_macos::StreamCaptureRegion { + rect_points: crate::state::RectPoints::new(60, 40, 220, 120), + rect_pixels: crate::state::RectPoints::new(120, 80, 440, 240), + }, + ); + + assert_eq!( + live_frame_stream_macos::stream_rect_for_requested_region( + capture_target, + crate::state::RectPoints::new(120, 80, 440, 240), + ), + Some(crate::state::RectPoints::new(0, 0, 440, 240)) + ); + assert_eq!( + live_frame_stream_macos::stream_rect_for_requested_region( + capture_target, + crate::state::RectPoints::new(140, 100, 100, 80), + ), + Some(crate::state::RectPoints::new(20, 20, 100, 80)) + ); + assert_eq!( + live_frame_stream_macos::stream_rect_for_requested_region( + capture_target, + crate::state::RectPoints::new(100, 80, 100, 80), + ), + None + ); + } + + #[test] + fn sample_handler_queue_label_is_monitor_scoped() { + assert_eq!( + live_frame_stream_macos::sample_handler_queue_label(7), + "io.hackink.rsnap.scroll-capture.sample-handler.monitor-7" + ); + assert_ne!( + live_frame_stream_macos::sample_handler_queue_label(7), + live_frame_stream_macos::sample_handler_queue_label(9) + ); + } + #[test] fn sample_cursor_from_bgra_bytes_reads_rgb_without_patch() { let sample = live_frame_stream_macos::sample_cursor_from_bgra_bytes( diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 39157fe5..25748700 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -1,8 +1,10 @@ mod hud_helpers; mod image_helpers; mod output; +pub(crate) mod replay_support; mod scroll_runtime; mod session_state; +mod trace_recording; mod window_runtime; #[cfg(target_os = "macos")] @@ -38,8 +40,8 @@ use egui::{ Layout, Margin, PointerButton, Pos2, Rect, Vec2, }; use egui::{ - Area, CentralPanel, ClippedPrimitive, Direction, Id, LayerId, Order, RichText, Sense, Shape, - Stroke, StrokeKind, UiBuilder, ViewportId, Visuals, + Area, CentralPanel, ClippedPrimitive, Id, LayerId, Order, RichText, Sense, Shape, Stroke, + StrokeKind, UiBuilder, ViewportId, Visuals, }; use egui_phosphor::{Variant, regular}; use egui_wgpu::{Renderer, ScreenDescriptor}; @@ -127,13 +129,16 @@ use self::session_state::{ LiveStreamStaleGrace, MacOSHudWindowConfigState, MacOSScrollPixelResidual, MacOSScrollWheelEvent, }; +use self::trace_recording::{ + ScrollCaptureTraceFrameRecord, ScrollCaptureTraceInputRecord, ScrollCaptureTraceRecorder, + ScrollCaptureTraceSessionSnapshot, +}; #[cfg(target_os = "macos")] use crate::backend; #[cfg(target_os = "macos")] use crate::live_frame_stream_macos::{CursorSampleRequest, MacLiveFrameStream}; use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; use crate::state::LiveCursorSample; -#[cfg(any(not(target_os = "macos"), test))] use crate::worker::CapturedMonitorRegionResult; use crate::{ state::{ @@ -214,6 +219,15 @@ const LIVE_WINDOW_LIST_REFRESH_INTERVAL: Duration = Duration::from_millis(120); const LIVE_PRESENT_INTERVAL_MIN: Duration = Duration::from_nanos(8_333_333); const HUD_LOUPE_MOVE_INTERVAL_MIN: Duration = LIVE_PRESENT_INTERVAL_MIN; const CURSOR_POLL_INTERVAL_MIN: Duration = LIVE_PRESENT_INTERVAL_MIN; +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_STREAM_EVENT_FALLBACK_POLL_INTERVAL: Duration = Duration::from_millis(40); +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_STREAM_POLL_INTERVAL: Duration = Duration::from_millis(8); +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_STREAM_BACKLOG_MAX_FRAMES: usize = 12; +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW: Duration = + Duration::from_millis(320); const OVERLAY_EVENT_LOOP_STALL_THRESHOLD: Duration = Duration::from_millis(250); #[cfg(target_os = "macos")] const SLOW_OP_WARN_CURSOR_LOCATION: Duration = Duration::from_millis(8); @@ -224,9 +238,17 @@ const SLOW_OP_WARN_RENDER: Duration = Duration::from_millis(24); const SLOW_OP_WARN_WINDOW_EVENT: Duration = Duration::from_millis(40); const SLOW_OP_WARN_INTERVAL: Duration = Duration::from_secs(1); const REDRAW_SUBSTEP_CONTRIBUTION_FLOOR: Duration = Duration::from_millis(4); -const SCROLL_CAPTURE_INPUT_FRESHNESS: Duration = Duration::from_millis(400); +// macOS trackpad/wheel sequences can keep delivering usable follow-up frames after the +// initiating input event. Keep the observation window wide enough for the capture pipeline +// to pair those frames before declaring the input stale. +const SCROLL_CAPTURE_INPUT_FRESHNESS: Duration = Duration::from_millis(600); +const SCROLL_CAPTURE_INPUT_MOTION_PRIOR_ROWS_MAX: f64 = 4_096.0; +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES: u8 = 5; #[cfg(target_os = "macos")] -const SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES: u8 = 3; +const SCROLL_CAPTURE_DUPLICATE_STREAM_STALL_THRESHOLD: u8 = 3; +#[cfg(target_os = "macos")] +const SCROLL_CAPTURE_DUPLICATE_STREAM_REFRESH_INTERVAL: Duration = Duration::from_millis(80); const HUD_PILL_INNER_MARGIN_X_POINTS: f32 = 12.0; const HUD_PILL_INNER_MARGIN_Y_POINTS: f32 = 8.0; const HUD_PILL_STROKE_WIDTH_POINTS: f32 = 1.0; @@ -280,8 +302,7 @@ const WINDOW_CAPTURE_MATTE_DARK_RGBA: image::Rgba = image::Rgba([24, 24, 24, const SCROLL_PREVIEW_WINDOW_WIDTH_POINTS: f64 = 260.0; const SCROLL_PREVIEW_WINDOW_HEIGHT_POINTS: f64 = 360.0; const SCROLL_PREVIEW_WINDOW_MARGIN_POINTS: i32 = 16; -#[cfg(not(target_os = "macos"))] -const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(50); +const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); #[cfg(target_os = "macos")] const SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE: Duration = Duration::from_millis(180); const SCROLL_CAPTURE_PREVIEW_WIDTH_PX: u32 = 320; @@ -438,7 +459,7 @@ impl FrozenToolbarTool { Self::Undo => "Undo", Self::Redo => "Redo", Self::AutoCenter => "Auto-center (C)", - Self::Scroll => "Scroll Capture ↓", + Self::Scroll => "Scroll Capture", Self::Copy => "Copy", Self::Save => "Save", } @@ -453,7 +474,7 @@ impl FrozenToolbarTool { Self::Undo => regular::ARROW_COUNTER_CLOCKWISE, Self::Redo => regular::ARROW_CLOCKWISE, Self::AutoCenter => regular::TARGET, - Self::Scroll => "↓", + Self::Scroll => regular::MOUSE_SCROLL, Self::Copy => regular::COPY, Self::Save => regular::FLOPPY_DISK, } @@ -470,15 +491,17 @@ impl FrozenToolbarTool { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ScrollCaptureFrameSource { - #[cfg(any(not(target_os = "macos"), test))] - Worker { request_id: u64 }, + Worker { + request_id: u64, + }, #[cfg(target_os = "macos")] - LiveStream { frame_seq: u64 }, + LiveStream { + frame_seq: u64, + }, } impl ScrollCaptureFrameSource { const fn as_str(self) -> &'static str { match self { - #[cfg(any(not(target_os = "macos"), test))] Self::Worker { .. } => "worker", #[cfg(target_os = "macos")] Self::LiveStream { .. } => "live_stream", @@ -487,7 +510,6 @@ impl ScrollCaptureFrameSource { const fn worker_request_id(self) -> Option { match self { - #[cfg(any(not(target_os = "macos"), test))] Self::Worker { request_id } => Some(request_id), #[cfg(target_os = "macos")] Self::LiveStream { .. } => None, @@ -982,13 +1004,30 @@ impl OverlaySession { )); if self.scroll_capture.active { - self.scroll_capture.live_stream = - Some(MacLiveFrameStream::with_self_capture_exception_window_ids_and_waker( + self.scroll_capture.live_stream = match ( + self.scroll_capture.capture_rect_points, + self.scroll_capture.capture_rect_pixels, + ) { + (Some(capture_rect_points), Some(capture_rect_pixels)) => { + Some(MacLiveFrameStream::with_scroll_capture_region_and_waker( + self.config.self_capture_exception_window_ids.clone(), + capture_rect_points, + capture_rect_pixels, + self.scroll_frame_waker.clone(), + )) + }, + _ => Some(MacLiveFrameStream::with_self_capture_exception_window_ids_and_waker( self.config.self_capture_exception_window_ids.clone(), self.scroll_frame_waker.clone(), - )); + )), + }; self.scroll_capture.last_stream_frame_seq = 0; + self.scroll_capture.last_stream_frame_fingerprint = None; + self.scroll_capture.consecutive_identical_stream_frames = 0; + self.scroll_capture.last_consumed_stream_frame_captured_at = None; + self.scroll_capture.pending_post_stall_burst_after_seq = None; self.scroll_capture.live_stream_stale_grace = None; + self.scroll_capture.last_duplicate_stream_refresh_at = None; } self.refresh_active_worker_for_self_capture_exception_window_ids_if_safe(); @@ -1452,6 +1491,17 @@ impl OverlaySession { self.repaint_interval_for_monitor(monitor) } + /// Returns the active repaint cadence that keeps interactive overlays responsive. + pub fn interactive_wait_interval(&self) -> Duration { + let monitor = if self.scroll_capture.active { + self.scroll_capture.monitor.or(self.state.monitor) + } else { + self.active_cursor_monitor() + }; + + self.repaint_interval_for_monitor(monitor) + } + fn live_sample_request_pending(&self) -> bool { self.latest_live_cursor_sample_request_id.is_some() && self.applied_live_cursor_sample_request_id @@ -1911,12 +1961,13 @@ impl OverlaySession { self.observe_scroll_capture_frame_at(frame, Instant::now()) } + #[cfg(test)] fn observe_scroll_capture_frame_at( &mut self, frame: RgbaImage, observation_at: Instant, ) -> Option> { - self.observe_scroll_capture_frame_with_gate(frame, false, observation_at) + self.observe_scroll_capture_frame_with_gate(frame, false, observation_at, false) } fn observe_scroll_capture_frame_with_gate( @@ -1924,14 +1975,20 @@ impl OverlaySession { frame: RgbaImage, allow_stale_input: bool, observation_at: Instant, + allow_post_stall_burst_search: bool, ) -> Option> { - if let Some(reason) = self.scroll_capture_observation_block_reason_at(observation_at) - && !(allow_stale_input && reason == "stale_input") - { - return None; + let prior_block_reason = self.scroll_capture_observation_block_reason_at(observation_at); + #[cfg(target_os = "macos")] + let consumed_live_stream_stale_grace = !allow_stale_input + && prior_block_reason == Some("stale_input") + && self.consume_live_stream_stale_grace_if_current(); + #[cfg(not(target_os = "macos"))] + let consumed_live_stream_stale_grace = false; + let allow_gate_bypass = allow_stale_input || consumed_live_stream_stale_grace; + let motion_rows_hint = self.scroll_capture_commit_motion_rows_hint_at(observation_at); + if !allow_gate_bypass && prior_block_reason.is_some() { + return Some(Ok(ScrollObserveOutcome::NoChange)); } - - let direction = self.scroll_capture.input_direction?; let result = { let Some(session) = self.scroll_capture.session.as_mut() else { self.scroll_capture_set_error("Scroll capture session is unavailable."); @@ -1939,22 +1996,114 @@ impl OverlaySession { return None; }; - match direction { - ScrollDirection::Down => session.observe_downward_sample(frame), - ScrollDirection::Up => session.observe_upward_sample(frame), - } + session.observe_downward_sample_with_motion_hint_and_burst( + frame, + motion_rows_hint, + allow_post_stall_burst_search, + ) }; + if let Ok(outcome) = &result { + self.consume_scroll_capture_downward_motion_rows_for_outcome(outcome); + } Some(result) } + fn scroll_capture_commit_motion_rows_hint_at(&self, observation_at: Instant) -> Option { + if self.scroll_capture.input_direction != Some(ScrollDirection::Down) { + return None; + } + + let Some(input_direction_at) = self.scroll_capture.input_direction_at else { + return None; + }; + if !self.scroll_capture.input_gesture_active + && observation_at.saturating_duration_since(input_direction_at) + > SCROLL_CAPTURE_INPUT_FRESHNESS + { + return None; + } + if !self.scroll_capture.downward_motion_rows_pending.is_finite() + || self.scroll_capture.downward_motion_rows_pending <= 0.0 + { + return None; + } + + Some(self.scroll_capture.downward_motion_rows_pending.ceil() as u32) + } + fn sync_scroll_preview_segments(&mut self) { - if let Some(preview) = self.scroll_preview_window.as_mut() { - let image = self.scroll_capture.session.as_ref().map(ScrollSession::preview_image); + let image = self.current_scroll_preview_render_image(); + { + let Some(preview) = self.scroll_preview_window.as_mut() else { + return; + }; preview.sync_image(image); preview.window.request_redraw(); } + + if let Some(monitor) = self.scroll_capture.monitor.or(self.state.monitor) { + self.position_scroll_preview_window(monitor); + } + } + + fn refresh_scroll_preview_committed_image(&mut self) { + self.scroll_capture.preview_committed_image = + self.scroll_capture.session.as_ref().map(|session| session.export_image().clone()); + } + + fn refresh_scroll_preview_display_image(&mut self) { + let motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_motion_rows_hint = motion_rows_hint; + self.scroll_capture.last_overlay_preview_provisional_motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_existing_candidate_height = None; + self.scroll_capture.last_overlay_preview_existing_candidate_motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_ledger_candidate_height = None; + self.scroll_capture.last_overlay_preview_ledger_candidate_motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_retained_candidate_height = None; + self.scroll_capture.last_overlay_preview_retained_candidate_motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_retained_hint_matches_motion_rows = false; + self.scroll_capture.last_overlay_preview_fresh_latest_frame_can_drive = false; + self.scroll_capture.last_overlay_preview_strong_unresolved_registration = false; + self.scroll_capture.last_overlay_preview_latest_frame_present = + self.scroll_capture.preview_latest_frame.is_some(); + self.scroll_capture.last_overlay_preview_used_provisional = false; + if let Some(session) = self.scroll_capture.session.as_mut() { + self.scroll_capture.preview_committed_image = Some(session.export_image().clone()); + self.scroll_capture.preview_display_image = + self.scroll_capture.preview_committed_image.clone(); + + return; + } + + self.scroll_capture.preview_display_image = + self.scroll_capture.preview_committed_image.as_ref().map(|base_preview| { + crate::scroll_capture::compose_provisional_preview_image( + base_preview, + self.scroll_capture.preview_latest_frame.as_ref(), + motion_rows_hint, + SCROLL_CAPTURE_PREVIEW_WIDTH_PX, + ) + }); + } + + fn scroll_capture_preview_dimensions(&self) -> Option<[u32; 2]> { + self.current_scroll_preview_render_image() + .as_ref() + .map(|image| [image.width(), image.height()]) + } + + fn scroll_preview_display_size_points(&self) -> Option { + let [width_px, height_px] = self.scroll_capture_preview_dimensions()?; + if width_px == 0 || height_px == 0 { + return None; + } + + let width_points = SCROLL_PREVIEW_WINDOW_WIDTH_POINTS as f32; + let scale = width_points / width_px as f32; + + Some(Vec2::new(width_points, (height_px as f32 * scale).max(1.0))) } fn scroll_capture_set_error(&mut self, message: impl Into) { @@ -1966,6 +2115,10 @@ impl OverlaySession { "Scroll capture paused on error." ); + if let Some(trace_recorder) = self.scroll_capture.trace_recorder.as_mut() { + trace_recorder.record_error(&message); + } + self.scroll_capture.paused = true; self.state.set_error(message); @@ -2007,7 +2160,6 @@ impl OverlaySession { } } - #[cfg(any(not(target_os = "macos"), test))] while let Some(resp) = self.worker.as_ref().and_then(|worker| worker.try_recv_captured_monitor_region()) { @@ -2491,7 +2643,6 @@ impl OverlaySession { self.png_encode_inflight = false; } }, - #[cfg(any(not(target_os = "macos"), test))] WorkerErrorSource::CaptureMonitorRegion => {}, } @@ -3593,7 +3744,7 @@ impl OverlaySession { state: ElementState::Pressed, button: MouseButton::Right, .. - } => self.exit(OverlayExit::Cancelled), + } => self.cancel_overlay("scroll_preview_right_click"), WindowEvent::KeyboardInput { event, .. } => self.handle_key_event(event), WindowEvent::ModifiersChanged(modifiers) => { self.handle_modifiers_changed(modifiers) @@ -3607,12 +3758,12 @@ impl OverlaySession { .as_ref() .is_some_and(|toolbar_window| toolbar_window.window.id() == window_id); let control = match event { - WindowEvent::CloseRequested => self.exit(OverlayExit::Cancelled), + WindowEvent::CloseRequested => self.cancel_overlay("window_close_requested"), WindowEvent::MouseInput { state: ElementState::Pressed, button: MouseButton::Right, .. - } => self.exit(OverlayExit::Cancelled), + } => self.cancel_overlay("window_right_click"), WindowEvent::Resized(size) if toolbar_window_id => { self.handle_toolbar_window_resized(*size) }, @@ -5017,6 +5168,66 @@ impl OverlaySession { } } + fn scroll_capture_direction_from_external_input_delta_y( + delta_y: f64, + ) -> Option { + if delta_y == 0.0 { + return None; + } + + Self::scroll_capture_direction_from_delta_y(delta_y) + } + + fn scroll_capture_motion_rows_from_wheel_delta(delta: &MouseScrollDelta) -> f64 { + match delta { + MouseScrollDelta::LineDelta(_, y) => f64::from(*y).abs(), + MouseScrollDelta::PixelDelta(delta) => { + #[cfg(target_os = "macos")] + { + Self::normalize_macos_scroll_pixel_component(delta.y).abs() + } + #[cfg(not(target_os = "macos"))] + { + delta.y.abs() + } + }, + } + } + + fn accumulate_scroll_capture_downward_motion_rows(&mut self, motion_rows: f64) { + if !motion_rows.is_finite() || motion_rows <= 0.0 { + return; + } + + self.scroll_capture.downward_motion_rows_pending = + (self.scroll_capture.downward_motion_rows_pending + motion_rows.abs()) + .clamp(0.0, SCROLL_CAPTURE_INPUT_MOTION_PRIOR_ROWS_MAX); + } + + fn clear_scroll_capture_downward_motion_rows(&mut self) { + self.scroll_capture.downward_motion_rows_pending = 0.0; + } + + fn consume_scroll_capture_downward_motion_rows(&mut self, consumed_rows: u32) { + if consumed_rows == 0 { + return; + } + + let remaining = self.scroll_capture.downward_motion_rows_pending - f64::from(consumed_rows); + self.scroll_capture.downward_motion_rows_pending = remaining.max(0.0); + } + + fn consume_scroll_capture_downward_motion_rows_for_outcome( + &mut self, + outcome: &ScrollObserveOutcome, + ) { + if let ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows } = + outcome + { + self.consume_scroll_capture_downward_motion_rows(*growth_rows); + } + } + fn record_scroll_capture_input_direction_at( &mut self, direction: ScrollDirection, @@ -5038,6 +5249,14 @@ impl OverlaySession { ) { if let Some(direction) = Self::scroll_capture_direction_from_wheel_delta(delta) { self.record_scroll_capture_input_direction_at(direction, false, at); + + if matches!(direction, ScrollDirection::Down) { + self.accumulate_scroll_capture_downward_motion_rows( + Self::scroll_capture_motion_rows_from_wheel_delta(delta), + ); + } else { + self.clear_scroll_capture_downward_motion_rows(); + } } } @@ -5061,8 +5280,27 @@ impl OverlaySession { gesture_ended: bool, at: Instant, ) { - if let Some(direction) = Self::scroll_capture_direction_from_delta_y(delta_y) { - self.record_scroll_capture_input_direction_at(direction, gesture_active, at); + if let Some(direction) = Self::scroll_capture_direction_from_external_input_delta_y(delta_y) + { + if self.should_absorb_upward_external_input_into_active_downward_gesture( + direction, + gesture_active, + ) { + self.record_scroll_capture_input_direction_at( + ScrollDirection::Down, + gesture_active, + at, + ); + self.accumulate_scroll_capture_downward_motion_rows(delta_y.abs()); + } else { + self.record_scroll_capture_input_direction_at(direction, gesture_active, at); + + if matches!(direction, ScrollDirection::Down) { + self.accumulate_scroll_capture_downward_motion_rows(delta_y.abs()); + } else { + self.clear_scroll_capture_downward_motion_rows(); + } + } } if gesture_ended { @@ -5070,6 +5308,17 @@ impl OverlaySession { } } + fn should_absorb_upward_external_input_into_active_downward_gesture( + &self, + direction: ScrollDirection, + gesture_active: bool, + ) -> bool { + gesture_active + && matches!(direction, ScrollDirection::Up) + && self.scroll_capture.input_direction == Some(ScrollDirection::Down) + && self.scroll_capture.downward_motion_rows_pending > 0.0 + } + fn apply_external_scroll_input_delta_y( &mut self, global_x: f64, @@ -5094,11 +5343,18 @@ impl OverlaySession { return; }; + #[cfg(not(target_os = "macos"))] if !capture_rect.contains(cursor_pixels) { return; } + #[cfg(target_os = "macos")] - if delta_y != 0.0 && !gesture_ended { + let _cursor_inside_capture_rect = capture_rect.contains(cursor_pixels); + #[cfg(target_os = "macos")] + if delta_y != 0.0 + && !gesture_ended + && !self.scroll_capture.overlay_mouse_passthrough_persistent + { self.arm_scroll_overlay_mouse_passthrough_window( Instant::now(), "external_scroll_input", @@ -5108,6 +5364,20 @@ impl OverlaySession { self.apply_scroll_capture_input_delta_y(delta_y, gesture_active, gesture_ended, at); } + fn scroll_capture_trace_snapshot_at( + &self, + observation_at: Instant, + ) -> ScrollCaptureTraceSessionSnapshot { + ScrollCaptureTraceSessionSnapshot::capture( + self.scroll_capture.session.as_ref(), + self.scroll_capture_preview_dimensions(), + self.scroll_capture.input_direction, + self.scroll_capture.input_gesture_active, + self.scroll_capture.downward_motion_rows_pending, + self.scroll_capture_input_age_ms_at(observation_at), + ) + } + #[cfg(test)] fn scroll_capture_input_allows_observation(&self) -> bool { self.scroll_capture_observation_block_reason().is_none() @@ -5115,10 +5385,6 @@ impl OverlaySession { #[cfg(test)] fn scroll_capture_input_allows_growth(&self) -> bool { - if self.scroll_capture.input_direction != Some(ScrollDirection::Down) { - return false; - } - self.scroll_capture_input_allows_observation() } @@ -5163,6 +5429,76 @@ impl OverlaySession { }) } + #[cfg(target_os = "macos")] + fn scroll_capture_should_force_stream_refresh_at(&self, now: Instant) -> bool { + if !self.scroll_capture_has_fresh_downward_backlog_at(now) { + return false; + } + + if self.scroll_capture.input_gesture_active { + return false; + } + + let Some(input_direction_at) = self.scroll_capture.input_direction_at else { + return false; + }; + + now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS + } + + #[cfg(target_os = "macos")] + fn scroll_capture_has_fresh_downward_backlog_at(&self, now: Instant) -> bool { + if self.scroll_capture.input_direction != Some(ScrollDirection::Down) + || self.scroll_capture.downward_motion_rows_pending <= 0.0 + { + return false; + } + + let Some(input_direction_at) = self.scroll_capture.input_direction_at else { + return false; + }; + + now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS + } + + #[cfg(target_os = "macos")] + fn scroll_capture_should_schedule_stale_stream_refresh_at(&self, now: Instant) -> bool { + if !self.scroll_capture.input_gesture_active { + return true; + } + + self.scroll_capture.last_stream_event_at.is_none_or(|last_stream_event_at| { + now.saturating_duration_since(last_stream_event_at) + >= SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + }) + } + + #[cfg(target_os = "macos")] + fn scroll_capture_should_allow_post_stall_burst_search_at( + &self, + frame_seq: u64, + now: Instant, + ) -> bool { + self.scroll_capture.pending_post_stall_burst_after_seq.is_some_and(|after_seq| { + frame_seq > after_seq && self.scroll_capture_has_fresh_downward_backlog_at(now) + }) + } + + #[cfg(target_os = "macos")] + fn scroll_capture_should_arm_post_stall_burst_for_time_gap_at( + &self, + frame_captured_at: Instant, + ) -> bool { + let Some(previous_captured_at) = self.scroll_capture.last_consumed_stream_frame_captured_at + else { + return false; + }; + + self.scroll_capture_has_fresh_downward_backlog_at(frame_captured_at) + && frame_captured_at.saturating_duration_since(previous_captured_at) + >= SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + } + fn toolbar_pointer_state( &mut self, monitor: MonitorRect, @@ -5217,7 +5553,7 @@ impl OverlaySession { } match &event.logical_key { - Key::Named(NamedKey::Escape) => self.exit(OverlayExit::Cancelled), + Key::Named(NamedKey::Escape) => self.cancel_overlay("escape_key"), Key::Named(NamedKey::Tab) => { let Some(rgb) = self.state.rgb else { return OverlayControl::Continue; @@ -5305,7 +5641,7 @@ impl OverlaySession { fn handle_scroll_capture_key_event(&mut self, event: &KeyEvent) -> OverlayControl { match &event.logical_key { - Key::Named(NamedKey::Escape) => self.exit(OverlayExit::Cancelled), + Key::Named(NamedKey::Escape) => self.cancel_overlay("scroll_capture_escape_key"), Key::Named(NamedKey::Space) => { self.begin_png_action(PngAction::Copy); @@ -5345,6 +5681,14 @@ impl OverlaySession { self.cropped_frozen_capture_image().or_else(|| self.state.frozen_image.clone()) } + fn current_scroll_preview_render_image(&self) -> Option { + if self.scroll_capture.active { + return self.current_export_image(); + } + + self.scroll_capture.preview_display_image.clone().or_else(|| self.current_export_image()) + } + fn scroll_capture_selection_is_ready(&self) -> bool { matches!(self.state.mode, OverlayMode::Frozen) && self.state.monitor.is_some() @@ -5453,20 +5797,36 @@ impl OverlaySession { fn build_scroll_capture_state( &self, monitor: MonitorRect, + capture_rect_points: RectPoints, capture_rect_pixels: RectPoints, base_frame: RgbaImage, ) -> Result { + let trace_recorder = ScrollCaptureTraceRecorder::from_env( + monitor, + capture_rect_pixels, + SCROLL_CAPTURE_PREVIEW_WIDTH_PX, + &base_frame, + ); + let preview_latest_frame = Some(base_frame.clone()); + let session = ScrollSession::new(base_frame, SCROLL_CAPTURE_PREVIEW_WIDTH_PX)?; + let preview_committed_image = Some(session.preview_image().clone()); + let preview_display_image = preview_committed_image.clone(); + Ok(ScrollCaptureState { active: true, paused: false, monitor: Some(monitor), + capture_rect_points: Some(capture_rect_points), capture_rect_pixels: Some(capture_rect_pixels), input_direction: None, input_direction_at: None, input_gesture_active: false, + downward_motion_rows_pending: 0.0, #[cfg(target_os = "macos")] overlay_mouse_passthrough_active: false, #[cfg(target_os = "macos")] + overlay_mouse_passthrough_persistent: false, + #[cfg(target_os = "macos")] overlay_mouse_passthrough_until: None, #[cfg(target_os = "macos")] external_scroll_input_drain_reader: self @@ -5478,24 +5838,59 @@ impl OverlaySession { #[cfg(target_os = "macos")] pixel_delta_residual: MacOSScrollPixelResidual::default(), #[cfg(target_os = "macos")] - live_stream: Some( - MacLiveFrameStream::with_self_capture_exception_window_ids_and_waker( - self.config.self_capture_exception_window_ids.clone(), - self.scroll_frame_waker.clone(), - ), - ), + live_stream: Some(MacLiveFrameStream::with_scroll_capture_region_and_waker( + self.config.self_capture_exception_window_ids.clone(), + capture_rect_points, + capture_rect_pixels, + self.scroll_frame_waker.clone(), + )), + #[cfg(target_os = "macos")] + live_stream_backlog: std::collections::VecDeque::new(), #[cfg(target_os = "macos")] last_stream_frame_seq: 0, #[cfg(target_os = "macos")] + last_stream_frame_fingerprint: None, + #[cfg(target_os = "macos")] + consecutive_identical_stream_frames: 0, + #[cfg(target_os = "macos")] + last_consumed_stream_frame_captured_at: None, + #[cfg(target_os = "macos")] + last_stream_event_at: None, + #[cfg(target_os = "macos")] + last_stream_poll_at: None, + #[cfg(target_os = "macos")] + last_duplicate_stream_refresh_at: None, + #[cfg(target_os = "macos")] + pending_post_stall_burst_after_seq: None, + #[cfg(target_os = "macos")] live_stream_stale_grace: None, - #[cfg(not(target_os = "macos"))] next_sample_at: Some(Instant::now() + SCROLL_CAPTURE_SAMPLE_INTERVAL), - #[cfg(not(target_os = "macos"))] next_request_id: 0, inflight_request_id: None, #[cfg(target_os = "macos")] inflight_request_observation: None, - session: Some(ScrollSession::new(base_frame, SCROLL_CAPTURE_PREVIEW_WIDTH_PX)?), + #[cfg(all(test, target_os = "macos"))] + force_worker_sampling_in_tests: false, + session: Some(session), + preview_committed_image, + preview_latest_frame, + preview_display_image, + retained_overlay_preview_image: None, + retained_overlay_preview_motion_rows_hint: None, + last_overlay_preview_motion_rows_hint: None, + last_overlay_preview_provisional_motion_rows_hint: None, + last_overlay_preview_existing_candidate_height: None, + last_overlay_preview_existing_candidate_motion_rows_hint: None, + last_overlay_preview_ledger_candidate_height: None, + last_overlay_preview_ledger_candidate_motion_rows_hint: None, + last_overlay_preview_retained_candidate_height: None, + last_overlay_preview_retained_candidate_motion_rows_hint: None, + last_overlay_preview_retained_hint_matches_motion_rows: false, + last_overlay_preview_fresh_latest_frame_can_drive: false, + last_overlay_preview_strong_unresolved_registration: false, + last_overlay_preview_latest_frame_present: false, + last_overlay_preview_used_provisional: false, + trace_recorder, }) } @@ -5555,22 +5950,33 @@ impl OverlaySession { } let base_frame_dimensions = base_frame.dimensions(); + self.scroll_capture = match self.build_scroll_capture_state( + monitor, + capture_rect_points, + capture_rect_pixels, + base_frame, + ) { + Ok(scroll_capture) => scroll_capture, + Err(err) => { + self.state.set_error(format!("{err:#}")); + self.request_redraw_all(); - self.scroll_capture = - match self.build_scroll_capture_state(monitor, capture_rect_pixels, base_frame) { - Ok(scroll_capture) => scroll_capture, - Err(err) => { - self.state.set_error(format!("{err:#}")); - self.request_redraw_all(); - - return OverlayControl::Continue; - }, - }; + return OverlayControl::Continue; + }, + }; if let Some(hook) = self.scroll_capture_started_hook.clone() { hook(); } + if let Some(trace_recorder) = self.scroll_capture.trace_recorder.as_ref() { + tracing::info!( + op = "scroll_capture.trace_recording_enabled", + manifest_path = %trace_recorder.manifest_path().display(), + "Enabled scroll-capture live trace recording for this session." + ); + } + tracing::info!( op = "scroll_capture.start", frozen_capture_source = ?self.frozen_capture_source, @@ -5585,10 +5991,12 @@ impl OverlaySession { ); self.sync_frozen_toolbar_state(); + self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_display_image(); self.sync_scroll_preview_segments(); self.position_scroll_preview_window(monitor); self.update_scroll_toolbar_default_position(monitor); - self.set_scroll_overlay_mouse_passthrough(false); + self.set_scroll_overlay_mouse_passthrough_persistent(true, "scroll_capture_started"); self.focus_scroll_keyboard_window(); if let Some(preview) = self.scroll_preview_window.as_ref() { @@ -5596,7 +6004,11 @@ impl OverlaySession { preview.window.request_redraw(); } - let _ = self.try_consume_scroll_stream_frame(); + if let (Some(monitor), Some(live_stream)) = + (self.scroll_capture.monitor, self.scroll_capture.live_stream.as_ref()) + { + live_stream.prime_monitor_nonblocking(monitor); + } self.request_redraw_for_monitor(monitor); @@ -5613,12 +6025,18 @@ impl OverlaySession { #[cfg(target_os = "macos")] if self.scroll_capture.paused { - self.disarm_scroll_overlay_mouse_passthrough(Instant::now(), "paused"); + self.set_scroll_overlay_mouse_passthrough_persistent(false, "paused"); } if !self.scroll_capture.paused { #[cfg(target_os = "macos")] { - let _ = self.try_consume_scroll_stream_frame(); + self.set_scroll_overlay_mouse_passthrough_persistent(true, "resumed"); + + if let (Some(monitor), Some(live_stream)) = + (self.scroll_capture.monitor, self.scroll_capture.live_stream.as_ref()) + { + live_stream.prime_monitor_nonblocking(monitor); + } } #[cfg(not(target_os = "macos"))] { @@ -5642,12 +6060,17 @@ impl OverlaySession { if !session.undo_last_append() { return; } + self.refresh_scroll_preview_committed_image(); self.clear_scroll_capture_inflight_request(); #[cfg(target_os = "macos")] { - let _ = self.try_consume_scroll_stream_frame(); + if let (Some(monitor), Some(live_stream)) = + (self.scroll_capture.monitor, self.scroll_capture.live_stream.as_ref()) + { + live_stream.prime_monitor_nonblocking(monitor); + } } #[cfg(not(target_os = "macos"))] { @@ -5655,6 +6078,7 @@ impl OverlaySession { Some(Instant::now() + SCROLL_CAPTURE_SAMPLE_INTERVAL); } + self.refresh_scroll_preview_display_image(); self.sync_scroll_preview_segments(); } @@ -5669,7 +6093,19 @@ impl OverlaySession { return; } - let Some(export_image) = self.current_export_image() else { + if self.scroll_capture.active { + self.maybe_tick_scroll_capture(); + self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + } + + let image = if self.scroll_capture.active { + self.current_scroll_preview_render_image() + } else { + self.current_export_image() + }; + let Some(export_image) = image else { return; }; @@ -6238,10 +6674,16 @@ impl OverlaySession { return; }; let preview_rect = self.scroll_preview_local_rect(monitor); - let _ = preview_window.window.request_inner_size(LogicalSize::new( - f64::from(preview_rect.width()), - f64::from(preview_rect.height()), - )); + let current_size = preview_window.window.inner_size(); + let desired_width = preview_rect.width().round().max(1.0) as u32; + let desired_height = preview_rect.height().round().max(1.0) as u32; + + if current_size.width != desired_width || current_size.height != desired_height { + let _ = preview_window.window.request_inner_size(LogicalSize::new( + f64::from(desired_width), + f64::from(desired_height), + )); + } preview_window.window.set_outer_position(LogicalPosition::new( f64::from(monitor.origin.x) + f64::from(preview_rect.min.x), @@ -6261,7 +6703,12 @@ impl OverlaySession { Vec2::new(capture_rect.width as f32, capture_rect.height as f32), ) .intersect(screen_rect); - let preview_height = capture_rect.height().max(1.0); + let preview_size = self + .scroll_preview_display_size_points() + .unwrap_or(Vec2::new(preview_width, capture_rect.height().max(1.0))); + let preview_width = preview_size.x.max(preview_width); + let max_preview_height = (screen_rect.max.y - capture_rect.min.y - gap).max(1.0); + let preview_height = preview_size.y.min(max_preview_height).max(1.0); let right_x = capture_rect.max.x + gap; let left_x = capture_rect.min.x - gap - preview_width; let x = if right_x + preview_width <= screen_rect.max.x { @@ -6338,9 +6785,27 @@ impl OverlaySession { } #[cfg(target_os = "macos")] - fn arm_scroll_overlay_mouse_passthrough_window(&mut self, now: Instant, reason: &'static str) { - let deadline = now + SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE; - let was_active = self.scroll_capture.overlay_mouse_passthrough_active; + fn set_scroll_overlay_mouse_passthrough_persistent( + &mut self, + passthrough: bool, + reason: &'static str, + ) { + let now = Instant::now(); + + self.scroll_capture.overlay_mouse_passthrough_persistent = passthrough; + self.scroll_capture.overlay_mouse_passthrough_until = None; + + self.set_scroll_overlay_mouse_passthrough_state(now, passthrough, reason); + } + + #[cfg(target_os = "macos")] + fn arm_scroll_overlay_mouse_passthrough_window(&mut self, now: Instant, reason: &'static str) { + if self.scroll_capture.overlay_mouse_passthrough_persistent { + return; + } + + let deadline = now + SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE; + let was_active = self.scroll_capture.overlay_mouse_passthrough_active; self.scroll_capture.overlay_mouse_passthrough_until = Some(deadline); @@ -6359,6 +6824,7 @@ impl OverlaySession { #[cfg(target_os = "macos")] fn disarm_scroll_overlay_mouse_passthrough(&mut self, now: Instant, reason: &'static str) { + self.scroll_capture.overlay_mouse_passthrough_persistent = false; self.scroll_capture.overlay_mouse_passthrough_until = None; self.set_scroll_overlay_mouse_passthrough_state(now, false, reason); @@ -6366,6 +6832,10 @@ impl OverlaySession { #[cfg(target_os = "macos")] fn sync_scroll_overlay_mouse_passthrough_window(&mut self, now: Instant) { + if self.scroll_capture.overlay_mouse_passthrough_persistent { + return; + } + if !self.scroll_capture.overlay_mouse_passthrough_active { return; } @@ -6760,7 +7230,63 @@ impl OverlaySession { } } + fn cancel_overlay(&mut self, reason: &'static str) -> OverlayControl { + tracing::info!( + op = "overlay.cancel_requested", + reason, + mode = ?self.state.mode, + scroll_capture_active = self.scroll_capture.active, + last_event_phase = %self.event_loop_phase.as_str(), + last_event_window_id = ?self.event_loop_last_progress_window_id, + last_event_monitor_id = ?self.event_loop_last_progress_monitor_id, + last_event_detail = ?self.event_loop_last_progress_detail, + "Overlay cancellation was requested." + ); + + self.exit(OverlayExit::Cancelled) + } + fn exit(&mut self, exit: OverlayExit) -> OverlayControl { + let (exit_kind, png_bytes_len, saved_path, error_message) = match &exit { + OverlayExit::Cancelled => ("cancelled", None, None, None), + OverlayExit::PngBytes(png_bytes) => ("png_bytes", Some(png_bytes.len()), None, None), + OverlayExit::Saved(path) => ("saved", None, Some(path.display().to_string()), None), + OverlayExit::Error(message) => ("error", None, None, Some(message.as_str())), + }; + tracing::info!( + op = "overlay.exit_begin", + exit_kind, + png_bytes_len, + saved_path, + error_message, + scroll_capture_active = self.scroll_capture.active, + scroll_capture_has_live_stream = self.scroll_capture.live_stream.is_some(), + live_sample_stream_present = self.live_sample_stream.is_some(), + last_event_phase = %self.event_loop_phase.as_str(), + last_event_window_id = ?self.event_loop_last_progress_window_id, + last_event_monitor_id = ?self.event_loop_last_progress_monitor_id, + last_event_detail = ?self.event_loop_last_progress_detail, + "Beginning overlay exit cleanup." + ); + if self.scroll_capture.active { + self.maybe_tick_scroll_capture(); + self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + } + let scroll_capture_final_snapshot = self.scroll_capture_trace_snapshot_at(Instant::now()); + let final_preview_image = self.current_scroll_preview_render_image(); + if let (Some(trace_recorder), Some(session)) = + (self.scroll_capture.trace_recorder.as_mut(), self.scroll_capture.session.as_ref()) + { + let final_preview_image = + final_preview_image.unwrap_or_else(|| session.preview_image().clone()); + trace_recorder.finalize_session( + session, + &final_preview_image, + scroll_capture_final_snapshot, + ); + } #[cfg(target_os = "macos")] self.set_scroll_overlay_mouse_passthrough(false); self.windows.clear(); @@ -6809,6 +7335,15 @@ impl OverlaySession { self.pending_png_action = None; self.keyboard_modifiers = ModifiersState::default(); + tracing::info!( + op = "overlay.exit_end", + exit_kind, + png_bytes_len, + saved_path, + error_message, + "Finished overlay exit cleanup." + ); + OverlayControl::Exit(exit) } @@ -7350,6 +7885,8 @@ struct FrozenToolbarButtonStyle { struct ScrollPreviewStrip { texture: TextureHandle, + pixel_size: [usize; 2], + rgba: Vec, size_points: Vec2, } @@ -7453,24 +7990,38 @@ impl ScrollPreviewWindow { self.window.request_redraw(); } - fn sync_image(&mut self, image: Option<&RgbaImage>) { - self.preview_image = image.map(|image| { - let preview_image = image_helpers::resize_scroll_preview_segment(image); - let color_image = ColorImage::from_rgba_unmultiplied( - [preview_image.width() as usize, preview_image.height() as usize], - preview_image.as_raw(), - ); - let texture = self.egui_ctx.load_texture( - String::from("scroll-preview-image"), - color_image, - TextureOptions::LINEAR, - ); - let ppp = self.window.scale_factor() as f32; - let size_points = - Vec2::new(preview_image.width() as f32 / ppp, preview_image.height() as f32 / ppp); + fn sync_image(&mut self, image: Option) { + let Some(image) = image else { + self.preview_image = None; - ScrollPreviewStrip { texture, size_points } - }); + return; + }; + let preview_image = image_helpers::resize_scroll_preview_segment(&image); + let pixel_size = [preview_image.width() as usize, preview_image.height() as usize]; + let rgba = preview_image.as_raw().clone(); + let color_image = ColorImage::from_rgba_unmultiplied(pixel_size, &rgba); + let ppp = self.window.scale_factor() as f32; + let size_points = + Vec2::new(preview_image.width() as f32 / ppp, preview_image.height() as f32 / ppp); + + match self.preview_image.as_mut() { + Some(strip) if strip.pixel_size == pixel_size => { + strip.texture.set(color_image, TextureOptions::LINEAR); + strip.pixel_size = pixel_size; + strip.rgba = rgba; + strip.size_points = size_points; + }, + _ => { + let texture = self.egui_ctx.load_texture( + String::from("scroll-preview-image"), + color_image, + TextureOptions::LINEAR, + ); + + self.preview_image = + Some(ScrollPreviewStrip { texture, pixel_size, rgba, size_points }); + }, + } } fn render_preview_ui(&mut self, view: ScrollPreviewView) -> FullOutput { @@ -7500,17 +8051,13 @@ impl ScrollPreviewWindow { if let Some(preview_image) = self.preview_image.as_ref() { let available = ui.available_size(); - let scale = (available.x / preview_image.size_points.x) - .min(available.y / preview_image.size_points.y) - .clamp(0.05, 1.0); + let scale = + (available.x / preview_image.size_points.x).clamp(0.05, 1.0); let draw_size = preview_image.size_points * scale; - ui.with_layout( - Layout::centered_and_justified(Direction::TopDown), - |ui| { - ui.image((preview_image.texture.id(), draw_size)); - }, - ); + ui.with_layout(Layout::top_down(egui::Align::Center), |ui| { + ui.image((preview_image.texture.id(), draw_size)); + }); } else { ui.allocate_space(ui.available_size()); } @@ -12408,6 +12955,8 @@ fn macos_configure_hud_window( #[cfg(test)] mod tests { + #[cfg(target_os = "macos")] + use std::collections::VecDeque; #[cfg(target_os = "macos")] use std::sync::Arc; #[cfg(target_os = "macos")] @@ -12424,16 +12973,21 @@ mod tests { #[cfg(target_os = "macos")] use crate::backend; #[cfg(target_os = "macos")] + use crate::backend::CaptureBackend; + #[cfg(target_os = "macos")] use crate::live_frame_stream_macos::MacLiveFrameStream; use crate::overlay::FrozenCaptureSource; use crate::overlay::PngAction; #[cfg(target_os = "macos")] + use crate::overlay::session_state::ScrollCaptureLiveFrame; + #[cfg(target_os = "macos")] use crate::overlay::{ AltActivationMode, HUD_PILL_CORNER_RADIUS_POINTS, HudPillGeometry, InflightScrollCaptureObservation, KCG_SCROLL_EVENT_UNIT_PIXEL, LiveSampleApplyResult, LiveStreamStaleGrace, MacOSScrollPixelResidual, OverlayControl, - SCROLL_CAPTURE_INPUT_FRESHNESS, SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, - SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE, ScrollCaptureFrameSource, StartupLiveRgbPlan, + SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW, SCROLL_CAPTURE_INPUT_FRESHNESS, + SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE, + ScrollCaptureFrameSource, StartupLiveRgbPlan, }; use crate::overlay::{ FrozenSelectionDragState, FrozenToolbarState, FrozenToolbarTool, @@ -12443,7 +12997,7 @@ mod tests { SELECTION_SIZE_BADGE_INSIDE_MARGIN_PX, SELECTION_SIZE_BADGE_SCREEN_MARGIN_PX, SelectionDashedBorderCache, SelectionDashedBorderMetrics, SelectionFlowGeometryCache, SelectionSizeBadgeTarget, TOOLBAR_CAPTURE_GAP_PX, TOOLBAR_SCREEN_MARGIN_PX, - ToolbarPlacement, Vec2, WindowRenderer, hud_helpers, + ToolbarPlacement, Vec2, WindowRenderer, hud_helpers, regular, }; use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; #[cfg(target_os = "macos")] @@ -12458,6 +13012,58 @@ mod tests { use crate::worker::OverlayWorker; use crate::worker::{WorkerErrorSource, WorkerResponse}; + #[cfg(target_os = "macos")] + struct SequenceScrollCaptureBackend { + frames: VecDeque>, + } + + #[cfg(target_os = "macos")] + impl SequenceScrollCaptureBackend { + fn new(frames: impl IntoIterator>) -> Self { + Self { frames: frames.into_iter().collect() } + } + } + + #[cfg(target_os = "macos")] + impl CaptureBackend for SequenceScrollCaptureBackend { + fn capture_monitor( + &mut self, + _monitor: MonitorRect, + ) -> color_eyre::eyre::Result { + Err(color_eyre::eyre::eyre!("unused in this test")) + } + + fn capture_monitor_region_for_scroll_capture( + &mut self, + _monitor: MonitorRect, + _rect_px: RectPoints, + ) -> color_eyre::eyre::Result> { + Ok(self.frames.pop_front().unwrap_or(None)) + } + + fn pixel_rgb_in_monitor( + &mut self, + _monitor: MonitorRect, + _point: GlobalPoint, + ) -> color_eyre::eyre::Result> { + Ok(None) + } + + fn rgba_patch_in_monitor( + &mut self, + _monitor: MonitorRect, + _point: GlobalPoint, + _width_px: u32, + _height_px: u32, + ) -> color_eyre::eyre::Result> { + Ok(None) + } + + fn refresh_window_cache(&mut self) -> color_eyre::eyre::Result> { + Err(color_eyre::eyre::eyre!("unused in this test")) + } + } + fn make_scroll_capture_test_image(width: u32, rows: &[[u8; 4]]) -> image::RgbaImage { let mut image = image::RgbaImage::new(width, rows.len() as u32); @@ -12479,12 +13085,115 @@ mod tests { make_scroll_capture_test_image(width, &document[start_row..start_row + window_rows]) } + #[cfg(target_os = "macos")] + fn make_sparse_worker_capture_window( + width: u32, + height: u32, + start_row: u32, + ) -> image::RgbaImage { + let stripe_x = 104_u32; + let mut image = image::RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255])); + + for y in 0..height { + let document_row = start_row.saturating_add(y); + let shade = ((document_row.saturating_mul(17)) % 180) as u8; + + for x in stripe_x..stripe_x.saturating_add(6) { + image.put_pixel(x, y, Rgba([shade, shade, shade, 255])); + } + for x in stripe_x.saturating_add(10)..stripe_x.saturating_add(13) { + if document_row % 19 < 9 { + image.put_pixel(x, y, Rgba([40, 40, 40, 255])); + } + } + } + + image + } + + #[cfg(target_os = "macos")] + fn make_browser_like_worker_capture_window( + width: u32, + height: u32, + start_row: u32, + ) -> image::RgbaImage { + let mut image = make_sparse_worker_capture_window(width, height, start_row); + let scrollbar_left = width.saturating_sub(18); + let content_left = 56_u32; + let content_right = width.saturating_sub(48); + let heading_width = 220_u32; + let paragraph_width = content_right.saturating_sub(content_left); + + for y in 0..height { + let document_row = start_row.saturating_add(y); + + if document_row % 420 < 18 { + for x in content_left..content_left.saturating_add(heading_width) { + image.put_pixel(x, y, Rgba([26, 26, 26, 255])); + } + } else if document_row % 420 >= 54 && document_row % 420 < 220 { + if document_row % 24 < 3 { + let trim = ((document_row / 24) % 5) * 18; + for x in content_left + ..content_left.saturating_add(paragraph_width.saturating_sub(trim)) + { + image.put_pixel(x, y, Rgba([72, 72, 72, 255])); + } + } + } else if document_row % 420 >= 270 && document_row % 420 < 360 { + if document_row % 20 < 2 { + for x in content_left.saturating_add(20) + ..content_left.saturating_add(paragraph_width.saturating_sub(70)) + { + image.put_pixel(x, y, Rgba([98, 98, 98, 255])); + } + } + } + + for x in scrollbar_left..width { + image.put_pixel(x, y, Rgba([232, 232, 232, 255])); + } + } + + let thumb_height = (height / 5).max(16); + let thumb_top = (start_row / 3) % height.max(thumb_height + 1); + let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); + for y in thumb_top..thumb_top.saturating_add(thumb_height) { + for x in scrollbar_left.saturating_add(3)..width.saturating_sub(4) { + image.put_pixel(x, y, Rgba([96, 96, 96, 255])); + } + } + + image + } + fn set_scroll_capture_input(session: &mut OverlaySession, direction: ScrollDirection) { session.scroll_capture.input_direction = Some(direction); session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; } + #[cfg(target_os = "macos")] + fn enable_test_worker_scroll_capture_path(session: &mut OverlaySession) { + session.scroll_capture.force_worker_sampling_in_tests = true; + } + + #[cfg(target_os = "macos")] + fn drain_scroll_capture_worker_until_idle(session: &mut OverlaySession) { + for _ in 0..64 { + let _ = session.drain_worker_responses(); + if session.scroll_capture.inflight_request_id.is_none() { + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + + panic!( + "timed out waiting for worker scroll-capture response; inflight_request_id={:?}", + session.scroll_capture.inflight_request_id + ); + } + fn observe_scroll_capture_frame( session: &mut OverlaySession, frame: image::RgbaImage, @@ -14117,6 +14826,200 @@ mod tests { assert!(preview.max.x <= 760.0); } + #[test] + fn scroll_preview_grows_with_render_height_until_monitor_limit() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_400, + height: 900, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.state.frozen_capture_rect = Some(RectPoints::new(120, 160, 400, 320)); + session.scroll_capture.preview_display_image = Some(RgbaImage::new(320, 960)); + + let preview = session.scroll_preview_local_rect(monitor); + + assert_eq!(preview.min.y, 160.0); + assert_eq!(preview.height(), 724.0); + } + + #[test] + fn current_scroll_preview_render_image_prefers_committed_export_during_scroll_capture() { + let mut session = OverlaySession::new(); + let base = make_scroll_capture_test_image(3, &[[10, 0, 0, 255]; 8]); + let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); + let mismatched_preview = RgbaImage::from_pixel(320, 40, Rgba([99, 0, 0, 255])); + let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); + + let _ = scroll_session.observe_downward_sample(grown).expect("observe"); + let expected_export = scroll_session.export_image().clone(); + session.scroll_capture.active = true; + session.scroll_capture.session = Some(scroll_session); + session.scroll_capture.preview_display_image = Some(mismatched_preview.clone()); + + assert_eq!(session.current_scroll_preview_render_image().as_ref(), Some(&expected_export)); + } + + #[test] + fn current_scroll_preview_render_image_uses_preview_display_when_scroll_capture_is_inactive() { + let mut session = OverlaySession::new(); + let preview = RgbaImage::from_pixel(320, 64, Rgba([42, 0, 0, 255])); + + session.scroll_capture.preview_display_image = Some(preview.clone()); + + assert_eq!(session.current_scroll_preview_render_image().as_ref(), Some(&preview)); + } + + #[test] + fn scroll_capture_preview_dimensions_follow_render_authority_during_scroll_capture() { + let mut session = OverlaySession::new(); + let base = make_scroll_capture_test_image(3, &[[10, 0, 0, 255]; 8]); + let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); + let mismatched_preview = RgbaImage::from_pixel(320, 40, Rgba([99, 0, 0, 255])); + let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); + + let _ = scroll_session.observe_downward_sample(grown).expect("observe"); + let expected_export = scroll_session.export_image().clone(); + session.scroll_capture.active = true; + session.scroll_capture.session = Some(scroll_session); + session.scroll_capture.preview_display_image = Some(mismatched_preview.clone()); + + assert_eq!( + session.scroll_capture_preview_dimensions(), + Some([expected_export.width(), expected_export.height()]) + ); + } + + #[test] + fn refresh_scroll_preview_display_image_uses_export_sized_render_buffer_during_active_capture() + { + let mut session = OverlaySession::new(); + let base = make_scroll_capture_test_image(3, &[[10, 0, 0, 255]; 8]); + let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); + let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); + + let _ = scroll_session.observe_downward_sample(grown).expect("observe"); + let expected_committed = scroll_session.export_image().clone(); + let expected_render = scroll_session.export_image().clone(); + + session.scroll_capture.active = true; + session.scroll_capture.session = Some(scroll_session); + session.refresh_scroll_preview_committed_image(); + session.refresh_scroll_preview_display_image(); + + assert_eq!( + session.scroll_capture.preview_committed_image.as_ref(), + Some(&expected_committed) + ); + assert_eq!(session.scroll_capture.preview_display_image.as_ref(), Some(&expected_render)); + assert_eq!(session.scroll_capture.last_overlay_preview_provisional_motion_rows_hint, None); + assert_eq!(session.scroll_capture.last_overlay_preview_existing_candidate_height, None); + assert_eq!( + session.scroll_capture.last_overlay_preview_existing_candidate_motion_rows_hint, + None + ); + assert_eq!(session.scroll_capture.last_overlay_preview_ledger_candidate_height, None); + assert_eq!( + session.scroll_capture.last_overlay_preview_ledger_candidate_motion_rows_hint, + None + ); + assert_eq!(session.scroll_capture.last_overlay_preview_retained_candidate_height, None); + assert_eq!( + session.scroll_capture.last_overlay_preview_retained_candidate_motion_rows_hint, + None + ); + assert!(!session.scroll_capture.last_overlay_preview_retained_hint_matches_motion_rows); + assert!(!session.scroll_capture.last_overlay_preview_fresh_latest_frame_can_drive); + assert!(!session.scroll_capture.last_overlay_preview_strong_unresolved_registration); + assert!(!session.scroll_capture.last_overlay_preview_latest_frame_present); + assert!(!session.scroll_capture.last_overlay_preview_used_provisional); + assert_eq!( + session.scroll_capture_preview_dimensions(), + Some([expected_render.width(), expected_render.height()]) + ); + } + + #[test] + fn begin_png_action_copies_preview_render_image_during_active_scroll_capture() { + let mut session = OverlaySession::new(); + let base = make_scroll_capture_test_image(3, &[[10, 0, 0, 255]; 8]); + let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); + let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); + let _ = scroll_session.observe_downward_sample(grown).expect("observe"); + let expected_export = scroll_session.export_image().clone(); + + let monitor = test_monitor(); + + session.state.begin_freeze(monitor); + session.state.finish_freeze(monitor, test_frozen_image()); + session.state.frozen_capture_rect = Some(RectPoints::new(100, 120, 220, 180)); + session.frozen_capture_source = FrozenCaptureSource::DragRegion; + session.authoritative_frozen_capture_ready = true; + session.scroll_capture.active = true; + session.scroll_capture.session = Some(scroll_session); + session.scroll_capture.preview_display_image = + Some(RgbaImage::from_pixel(320, 64, Rgba([77, 0, 0, 255]))); + + session.begin_png_action(PngAction::Copy); + + assert_eq!(session.pending_png_action, Some(PngAction::Copy)); + assert_eq!(session.pending_encode_png.as_ref(), Some(&expected_export)); + assert_eq!(session.state.error_message.as_deref(), Some("Copying...")); + } + + #[cfg(target_os = "macos")] + #[test] + fn duplicate_live_frames_schedule_forced_refresh_when_downward_backlog_is_fresh() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let observed_at = Instant::now(); + let frame = ScrollCaptureLiveFrame { + frame_seq: 7, + captured_at: observed_at, + image: RgbaImage::from_pixel(16, 16, Rgba([7, 8, 9, 255])), + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(observed_at); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(session.note_scroll_capture_live_stream_frame_activity(&frame)); + assert!(!session.note_scroll_capture_live_stream_frame_activity(&frame)); + assert!(!session.note_scroll_capture_live_stream_frame_activity(&frame)); + assert!(!session.note_scroll_capture_live_stream_frame_activity(&frame)); + assert_eq!(session.scroll_capture.consecutive_identical_stream_frames, 3); + + session.maybe_schedule_duplicate_stream_refresh(frame.frame_seq, observed_at); + + assert_eq!( + session + .scroll_capture + .live_stream + .as_ref() + .and_then(MacLiveFrameStream::debug_last_request_kind), + Some("refresh_monitor_nonblocking_if_stale") + ); + assert_eq!( + session.scroll_capture.pending_post_stall_burst_after_seq, + Some(frame.frame_seq) + ); + assert_eq!(session.scroll_capture.last_duplicate_stream_refresh_at, Some(observed_at)); + } + #[cfg(not(target_os = "macos"))] #[test] fn scroll_capture_is_unavailable_on_non_macos_even_with_drag_selection() { @@ -14262,13 +15165,29 @@ mod tests { assert_eq!(*hook_order, vec!["starting", "started"]); } + #[cfg(target_os = "macos")] + #[test] + fn scroll_capture_start_preserves_existing_live_sample_stream() { + let mut session = OverlaySession::new(); + + seed_ready_scroll_capture_selection(&mut session); + session.live_sample_stream = Some(MacLiveFrameStream::new()); + + let control = session.start_scroll_capture(); + + assert!(matches!(control, OverlayControl::Continue)); + assert!(session.scroll_capture.active); + assert!(session.live_sample_stream.is_some()); + assert!(session.scroll_capture.live_stream.is_some()); + } + #[cfg(target_os = "macos")] #[test] fn reset_for_start_preserves_external_scroll_input_drain_reader() { let mut session = OverlaySession::default(); session.set_external_scroll_input_drain_reader(Arc::new(|_, _| { - vec![(1, Instant::now(), 10.0, 20.0, -4.0, true, false)] + vec![(1, Instant::now(), 10.0, 20.0, 4.0, true, false)] })); session.reset_for_start(); @@ -14938,7 +15857,7 @@ mod tests { let start = Instant::now(); let events = Arc::new([ (1, start, 150.0, 160.0, -4.0, true, false), - (2, start + Duration::from_millis(2), 150.0, 160.0, 4.0, false, true), + (2, start + Duration::from_millis(2), 150.0, 160.0, -4.0, false, true), ]); let mut session = OverlaySession::new(); @@ -14969,7 +15888,7 @@ mod tests { session.drain_external_scroll_input_events_through(start + Duration::from_millis(2)); - assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); assert!(!session.scroll_capture.input_gesture_active); assert_eq!(session.scroll_capture.last_external_scroll_input_seq, 2); } @@ -15021,6 +15940,8 @@ mod tests { [40, 0, 0, 255], [50, 0, 0, 255], [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], ]; let monitor = MonitorRect { id: 1, @@ -15083,6 +16004,8 @@ mod tests { [40, 0, 0, 255], [50, 0, 0, 255], [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], ]; let monitor = MonitorRect { id: 1, @@ -15094,7 +16017,7 @@ mod tests { let capture_rect = RectPoints::new(100, 120, 200, 240); let through = Instant::now(); let events = - Arc::new([(7, through - Duration::from_millis(10), 150.0, 160.0, -4.0, false, false)]); + Arc::new([(7, through - Duration::from_millis(10), 150.0, 160.0, 4.0, false, false)]); let stale_at = through + SCROLL_CAPTURE_INPUT_FRESHNESS + Duration::from_millis(1); let mut session = OverlaySession::new(); @@ -15121,7 +16044,6 @@ mod tests { session.scroll_capture.live_stream_stale_grace, Some(LiveStreamStaleGrace { external_input_seq: 7, - input_direction: ScrollDirection::Down, remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, }) ); @@ -15133,30 +16055,17 @@ mod tests { ) .transpose() .unwrap(), - None - ); - - session.handle_scroll_capture_frame( - make_scroll_capture_window(&document, 3, 1, 5), - ScrollCaptureFrameSource::LiveStream { frame_seq: 143 }, - false, - stale_at, + Some(ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: 1, + }) ); - assert_eq!(scroll_capture_export_height(&session), 6); - assert_eq!( - session.scroll_capture.live_stream_stale_grace, - Some(LiveStreamStaleGrace { - external_input_seq: 7, - input_direction: ScrollDirection::Down, - remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES - 1, - }) - ); } #[cfg(target_os = "macos")] #[test] - fn stale_live_stream_grace_survives_same_direction_overlay_wheel_update() { + fn stale_live_stream_frame_is_observed_even_without_direction_freshness() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -15164,6 +16073,8 @@ mod tests { [40, 0, 0, 255], [50, 0, 0, 255], [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], ]; let monitor = MonitorRect { id: 1, @@ -15176,7 +16087,7 @@ mod tests { let through = Instant::now(); let wheel_at = through + Duration::from_millis(10); let events = - Arc::new([(7, through - Duration::from_millis(10), 150.0, 160.0, -4.0, false, false)]); + Arc::new([(7, through - Duration::from_millis(10), 150.0, 160.0, 4.0, false, false)]); let stale_at = wheel_at + SCROLL_CAPTURE_INPUT_FRESHNESS + Duration::from_millis(1); let mut session = OverlaySession::new(); @@ -15208,7 +16119,6 @@ mod tests { session.scroll_capture.live_stream_stale_grace, Some(LiveStreamStaleGrace { external_input_seq: 7, - input_direction: ScrollDirection::Down, remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, }) ); @@ -15220,30 +16130,17 @@ mod tests { ) .transpose() .unwrap(), - None - ); - - session.handle_scroll_capture_frame( - make_scroll_capture_window(&document, 3, 1, 5), - ScrollCaptureFrameSource::LiveStream { frame_seq: 143 }, - false, - stale_at, - ); - - assert_eq!(scroll_capture_export_height(&session), 6); - assert_eq!( - session.scroll_capture.live_stream_stale_grace, - Some(LiveStreamStaleGrace { - external_input_seq: 7, - input_direction: ScrollDirection::Down, - remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES - 1, + Some(ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: 1, }) ); + assert_eq!(scroll_capture_export_height(&session), 6); } #[cfg(target_os = "macos")] #[test] - fn live_stream_stale_grace_is_consumed_and_superseded() { + fn fresh_live_stream_frame_without_direction_metadata_fails_closed_as_no_change() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -15261,7 +16158,7 @@ mod tests { scale_factor_x1000: 1_000, }; let capture_rect = RectPoints::new(100, 120, 200, 240); - let stale_at = Instant::now() - Duration::from_millis(1); + let observed_at = Instant::now(); let mut session = OverlaySession::new(); session.scroll_capture.active = true; @@ -15269,58 +16166,15 @@ mod tests { session.scroll_capture.capture_rect_pixels = Some(capture_rect); session.scroll_capture.session = Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); - session.scroll_capture.last_external_scroll_input_seq = 7; - session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = - Some(stale_at - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(1)); - session.scroll_capture.input_gesture_active = false; - session.scroll_capture.live_stream_stale_grace = Some(LiveStreamStaleGrace { - external_input_seq: 7, - input_direction: ScrollDirection::Down, - remaining_stale_frames: 1, - }); session.handle_scroll_capture_frame( make_scroll_capture_window(&document, 3, 1, 5), ScrollCaptureFrameSource::LiveStream { frame_seq: 143 }, false, - stale_at, - ); - - assert_eq!(scroll_capture_export_height(&session), 6); - assert_eq!(session.scroll_capture.live_stream_stale_grace, None); - - let height_after_first_stale = scroll_capture_export_height(&session); - - session.handle_scroll_capture_frame( - make_scroll_capture_window(&document, 3, 2, 5), - ScrollCaptureFrameSource::LiveStream { frame_seq: 144 }, - false, - stale_at, - ); - - assert_eq!(scroll_capture_export_height(&session), height_after_first_stale); - - session.scroll_capture.last_external_scroll_input_seq = 8; - session.scroll_capture.input_direction = Some(ScrollDirection::Up); - session.scroll_capture.input_direction_at = - Some(stale_at - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(1)); - session.scroll_capture.input_gesture_active = false; - session.scroll_capture.live_stream_stale_grace = Some(LiveStreamStaleGrace { - external_input_seq: 7, - input_direction: ScrollDirection::Down, - remaining_stale_frames: 1, - }); - - session.handle_scroll_capture_frame( - make_scroll_capture_window(&document, 3, 1, 5), - ScrollCaptureFrameSource::LiveStream { frame_seq: 145 }, - false, - stale_at, + observed_at, ); - assert_eq!(scroll_capture_export_height(&session), height_after_first_stale); - assert_eq!(session.scroll_capture.live_stream_stale_grace, None); + assert_eq!(scroll_capture_export_height(&session), 5); } #[cfg(target_os = "macos")] @@ -15331,27 +16185,50 @@ mod tests { } #[test] - fn negative_vertical_wheel_delta_maps_to_downward_scroll_capture() { + fn positive_vertical_wheel_delta_maps_to_upward_scroll_capture() { assert_eq!( OverlaySession::scroll_capture_direction_from_wheel_delta( - &MouseScrollDelta::LineDelta(0.0, -1.0) + &MouseScrollDelta::LineDelta(0.0, 1.0) ), - Some(ScrollDirection::Down) + Some(ScrollDirection::Up) ); } #[test] - fn positive_vertical_wheel_delta_maps_to_upward_scroll_capture() { + fn negative_vertical_wheel_delta_maps_to_downward_scroll_capture() { assert_eq!( OverlaySession::scroll_capture_direction_from_wheel_delta( - &MouseScrollDelta::LineDelta(0.0, 1.0) + &MouseScrollDelta::LineDelta(0.0, -1.0) ), - Some(ScrollDirection::Up) + Some(ScrollDirection::Down) ); } #[test] - fn external_scroll_input_inside_capture_rect_updates_direction() { + fn external_scroll_input_inside_capture_rect_uses_upward_observation_for_positive_delta() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); + + session.handle_external_scroll_input_delta_y(150.0, 160.0, 4.0, true, false); + + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); + assert!(session.scroll_capture.input_direction_at.is_some()); + assert!(session.scroll_capture.input_gesture_active); + assert_eq!(session.scroll_capture.downward_motion_rows_pending, 0.0); + } + + #[test] + fn external_scroll_input_inside_capture_rect_uses_downward_observation_for_negative_delta() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -15373,7 +16250,30 @@ mod tests { } #[test] - fn external_scroll_input_outside_capture_rect_is_ignored() { + fn upward_external_scroll_input_clears_existing_downward_motion_backlog() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); + session.scroll_capture.downward_motion_rows_pending = 128.0; + + session.handle_external_scroll_input_delta_y(150.0, 160.0, 12.0, true, false); + + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); + assert_eq!(session.scroll_capture.downward_motion_rows_pending, 0.0); + } + + #[test] + #[cfg(target_os = "macos")] + fn external_scroll_input_outside_capture_rect_on_same_monitor_is_still_consumed() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -15389,10 +16289,56 @@ mod tests { session.handle_external_scroll_input_delta_y(50.0, 50.0, -4.0, true, false); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert!(session.scroll_capture.input_direction_at.is_some()); + assert!(session.scroll_capture.input_gesture_active); + assert_eq!(session.scroll_capture.downward_motion_rows_pending, 4.0); + } + + #[test] + #[cfg(not(target_os = "macos"))] + fn external_scroll_input_outside_capture_rect_is_ignored() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); + + session.handle_external_scroll_input_delta_y(50.0, 50.0, 4.0, true, false); + assert_eq!(session.scroll_capture.input_direction, None); assert!(session.scroll_capture.input_direction_at.is_none()); } + #[test] + fn external_scroll_input_outside_scroll_monitor_is_ignored() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(1_000, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); + + session.handle_external_scroll_input_delta_y(50.0, 50.0, 4.0, true, false); + + assert_eq!(session.scroll_capture.input_direction, None); + assert!(session.scroll_capture.input_direction_at.is_none()); + assert_eq!(session.scroll_capture.downward_motion_rows_pending, 0.0); + } + #[test] fn external_scroll_input_terminal_event_preserves_last_direction_for_freshness() { let monitor = MonitorRect { @@ -15449,6 +16395,44 @@ mod tests { assert!(session.scroll_capture.overlay_mouse_passthrough_until.is_none()); } + #[cfg(target_os = "macos")] + #[test] + fn scroll_capture_start_enables_persistent_passthrough() { + let mut session = OverlaySession::new(); + + seed_ready_scroll_capture_selection(&mut session); + + let control = session.start_scroll_capture(); + + assert!(matches!(control, OverlayControl::Continue)); + assert!(session.scroll_capture.active); + assert!(session.scroll_capture.overlay_mouse_passthrough_active); + assert!(session.scroll_capture.overlay_mouse_passthrough_persistent); + assert!(session.scroll_capture.overlay_mouse_passthrough_until.is_none()); + } + + #[cfg(target_os = "macos")] + #[test] + fn scroll_capture_pause_and_resume_toggle_persistent_passthrough() { + let mut session = OverlaySession::new(); + + seed_ready_scroll_capture_selection(&mut session); + let _ = session.start_scroll_capture(); + + session.toggle_scroll_capture_paused(); + + assert!(session.scroll_capture.paused); + assert!(!session.scroll_capture.overlay_mouse_passthrough_active); + assert!(!session.scroll_capture.overlay_mouse_passthrough_persistent); + + session.toggle_scroll_capture_paused(); + + assert!(!session.scroll_capture.paused); + assert!(session.scroll_capture.overlay_mouse_passthrough_active); + assert!(session.scroll_capture.overlay_mouse_passthrough_persistent); + assert!(session.scroll_capture.overlay_mouse_passthrough_until.is_none()); + } + #[cfg(target_os = "macos")] #[test] fn external_scroll_input_extends_passthrough_window_inside_capture_rect() { @@ -15470,14 +16454,14 @@ mod tests { let first_deadline = session.scroll_capture.overlay_mouse_passthrough_until; - session.handle_external_scroll_input_delta_y(150.0, 160.0, -4.0, true, false); + session.handle_external_scroll_input_delta_y(150.0, 160.0, 4.0, true, false); assert!(session.scroll_capture.overlay_mouse_passthrough_active); assert!(session.scroll_capture.overlay_mouse_passthrough_until > first_deadline); } #[test] - fn terminal_downward_scroll_event_sets_direction_before_finishing() { + fn terminal_positive_scroll_event_sets_upward_observation_before_finishing() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -15491,16 +16475,16 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); - session.handle_external_scroll_input_delta_y(150.0, 160.0, -4.0, false, true); + session.handle_external_scroll_input_delta_y(150.0, 160.0, 4.0, false, true); - assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); assert!(session.scroll_capture.input_direction_at.is_some()); assert!(!session.scroll_capture.input_gesture_active); assert!(session.scroll_capture_input_allows_growth()); } #[test] - fn terminal_upward_scroll_event_does_not_allow_growth() { + fn terminal_negative_scroll_event_still_allows_downward_growth() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -15514,12 +16498,12 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); - session.handle_external_scroll_input_delta_y(150.0, 160.0, 4.0, false, true); + session.handle_external_scroll_input_delta_y(150.0, 160.0, -4.0, false, true); - assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); assert!(session.scroll_capture.input_direction_at.is_some()); assert!(!session.scroll_capture.input_gesture_active); - assert!(!session.scroll_capture_input_allows_growth()); + assert!(session.scroll_capture_input_allows_growth()); } #[cfg(target_os = "macos")] @@ -15532,11 +16516,11 @@ mod tests { session.set_external_scroll_input_drain_reader(Arc::new(|_, _| Vec::new())); session.record_scroll_capture_input_direction_from_overlay_wheel_at( - &MouseScrollDelta::LineDelta(0.0, -1.0), + &MouseScrollDelta::LineDelta(0.0, 1.0), observed_at, ); - assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Up)); assert_eq!(session.scroll_capture.input_direction_at, Some(observed_at)); assert!(!session.scroll_capture.input_gesture_active); } @@ -15560,7 +16544,7 @@ mod tests { session.scroll_capture.input_gesture_active = true; assert!(session.scroll_capture_input_allows_observation()); - assert!(!session.scroll_capture_input_allows_growth()); + assert!(session.scroll_capture_input_allows_growth()); } #[test] @@ -15576,7 +16560,7 @@ mod tests { } #[test] - fn upward_direction_never_allows_growth() { + fn upward_direction_still_allows_growth_gate() { let mut session = OverlaySession::new(); session.scroll_capture.active = true; @@ -15584,11 +16568,11 @@ mod tests { session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; - assert!(!session.scroll_capture_input_allows_growth()); + assert!(session.scroll_capture_input_allows_growth()); } #[test] - fn upward_rewind_frame_is_observed_before_resume_frontier_growth() { + fn upward_input_does_not_dirty_later_downward_growth() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -15646,13 +16630,13 @@ mod tests { set_scroll_capture_input(&mut session, ScrollDirection::Down); - assert!(matches!( + assert_eq!( observe_scroll_capture_frame( &mut session, make_scroll_capture_window(&document, 3, 2, 5), ), - Some(ScrollObserveOutcome::NoChange) | Some(ScrollObserveOutcome::PreviewUpdated) - )); + Some(ScrollObserveOutcome::NoChange) + ); assert_eq!(scroll_capture_export_height(&session), height_after_second_append); set_scroll_capture_input(&mut session, ScrollDirection::Up); @@ -15672,13 +16656,13 @@ mod tests { set_scroll_capture_input(&mut session, ScrollDirection::Down); - assert!(matches!( + assert_eq!( observe_scroll_capture_frame( &mut session, make_scroll_capture_window(&document, 3, 2, 5), ), - Some(ScrollObserveOutcome::NoChange) | Some(ScrollObserveOutcome::PreviewUpdated) - )); + Some(ScrollObserveOutcome::NoChange) + ); assert_eq!(scroll_capture_export_height(&session), height_after_second_append); assert_eq!( observe_scroll_capture_frame( @@ -15694,7 +16678,7 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn stale_latched_worker_input_rewinds_without_ax_position() { + fn stale_latched_worker_input_fails_closed_without_appending_growth() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -15756,9 +16740,9 @@ mod tests { session.scroll_capture.inflight_request_id = Some(41); session.scroll_capture.inflight_request_observation = Some(InflightScrollCaptureObservation { - input_direction: Some(ScrollDirection::Up), was_observable: true, external_input_seq: 7, + input_direction: Some(ScrollDirection::Down), }); session.handle_captured_scroll_region( @@ -15775,11 +16759,11 @@ mod tests { format!("{:?}", session.scroll_capture.session.as_ref().unwrap()); assert!( - scroll_session_debug.contains("resume_frontier_top_y: Some(2)"), + scroll_session_debug.contains("resume_frontier_top_y: None"), "{scroll_session_debug}" ); assert!( - scroll_session_debug.contains("observed_viewport_top_y: 1"), + scroll_session_debug.contains("observed_viewport_top_y: 2"), "{scroll_session_debug}" ); assert_eq!( @@ -15790,17 +16774,7 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn newer_input_supersedes_latched_worker_observation_context() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - ]; + fn newer_same_direction_input_keeps_latched_worker_observation_context() { let monitor = MonitorRect { id: 1, origin: GlobalPoint::new(0, 0), @@ -15810,92 +16784,1030 @@ mod tests { }; let capture_rect = RectPoints::new(100, 120, 200, 240); let mut session = OverlaySession::new(); + let base = make_sparse_worker_capture_window(512, 640, 0); + let next = make_sparse_worker_capture_window(512, 640, 90); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(capture_rect); - session.scroll_capture.session = - Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(Instant::now()); - session.scroll_capture.input_gesture_active = true; - - assert_eq!( - session - .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 1, 5)) - .transpose() - .unwrap(), - Some(ScrollObserveOutcome::Committed { - direction: ScrollDirection::Down, - growth_rows: 1, - }) - ); - assert_eq!( - session - .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 2, 5)) - .transpose() - .unwrap(), - Some(ScrollObserveOutcome::Committed { - direction: ScrollDirection::Down, - growth_rows: 1, - }) - ); + session.scroll_capture.input_gesture_active = false; - let height_after_second_append = + let height_before_worker_frame = session.scroll_capture.session.as_ref().unwrap().export_image().height(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = - Some(Instant::now() - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(50)); - session.scroll_capture.input_gesture_active = false; + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; session.scroll_capture.last_external_scroll_input_seq = 8; session.scroll_capture.inflight_request_id = Some(41); session.scroll_capture.inflight_request_observation = Some(InflightScrollCaptureObservation { - input_direction: Some(ScrollDirection::Up), was_observable: true, external_input_seq: 7, + input_direction: Some(ScrollDirection::Down), }); - session.handle_captured_scroll_region( - monitor, - capture_rect, - 41, - make_scroll_capture_window(&document, 3, 1, 5), - ); + session.handle_captured_scroll_region(monitor, capture_rect, 41, next); assert_eq!(session.scroll_capture.inflight_request_id, None); assert_eq!(session.scroll_capture.inflight_request_observation, None); - - let scroll_session_debug = - format!("{:?}", session.scroll_capture.session.as_ref().unwrap()); - - assert!(scroll_session_debug.contains("resume_frontier_top_y: None")); - assert!(scroll_session_debug.contains("current_viewport_top_y: 2")); assert_eq!( session.scroll_capture.session.as_ref().unwrap().export_image().height(), - height_after_second_append + height_before_worker_frame + 90 ); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 90); + } + + #[cfg(target_os = "macos")] + #[test] + fn stale_same_direction_worker_frame_keeps_latched_worker_observation_context() { + let monitor = test_monitor(); + let capture_rect = RectPoints::new(100, 120, 512, 640); + let mut session = OverlaySession::new(); + let base = make_sparse_worker_capture_window(512, 640, 0); + let next = make_sparse_worker_capture_window(512, 640, 90); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = + Some(Instant::now() - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(50)); + session.scroll_capture.input_gesture_active = false; + session.scroll_capture.last_external_scroll_input_seq = 8; + session.scroll_capture.inflight_request_id = Some(41); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: 7, + input_direction: Some(ScrollDirection::Down), + }); + + session.handle_captured_scroll_region(monitor, capture_rect, 41, next); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().export_image().height(), 730); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 90); + } + + #[cfg(target_os = "macos")] + #[test] + fn newer_opposite_direction_supersedes_latched_worker_observation_context() { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + ]; + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = + Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + + assert_eq!( + session + .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 1, 5)) + .transpose() + .unwrap(), + Some(ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: 1, + }) + ); + assert_eq!( + session + .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 2, 5)) + .transpose() + .unwrap(), + Some(ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: 1, + }) + ); + + let height_after_second_append = + session.scroll_capture.session.as_ref().unwrap().export_image().height(); + + session.scroll_capture.input_direction = Some(ScrollDirection::Up); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_external_scroll_input_seq = 8; + session.scroll_capture.inflight_request_id = Some(41); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: 7, + input_direction: Some(ScrollDirection::Down), + }); + + session.handle_captured_scroll_region( + monitor, + capture_rect, + 41, + make_scroll_capture_window(&document, 3, 3, 5), + ); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + height_after_second_append + ); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 2); + } + + #[cfg(target_os = "macos")] + #[test] + fn successive_same_direction_worker_frames_do_not_stall_after_newer_input() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = + Some(ScrollSession::new(make_sparse_worker_capture_window(512, 640, 0), 320).unwrap()); + + for (step, start_row) in [90_u32, 180, 270].into_iter().enumerate() { + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 2; + session.scroll_capture.inflight_request_id = Some(41 + step as u64); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: (step as u64) + 1, + input_direction: Some(ScrollDirection::Down), + }); + + session.handle_captured_scroll_region( + monitor, + capture_rect, + 41 + step as u64, + make_sparse_worker_capture_window(512, 640, start_row), + ); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), + start_row as i32 + ); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + 640 + start_row + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn successive_browser_like_worker_frames_do_not_stall_after_newer_input() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = Some( + ScrollSession::new(make_browser_like_worker_capture_window(512, 640, 0), 320).unwrap(), + ); + + for (step, start_row) in [84_u32, 168, 252].into_iter().enumerate() { + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 12; + session.scroll_capture.inflight_request_id = Some(81 + step as u64); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: (step as u64) + 11, + input_direction: Some(ScrollDirection::Down), + }); + + session.handle_captured_scroll_region( + monitor, + capture_rect, + 81 + step as u64, + make_browser_like_worker_capture_window(512, 640, start_row), + ); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), + start_row as i32 + ); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + 640 + start_row + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn missing_worker_scroll_frame_clears_inflight_without_mutating_session() { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + ]; + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = + Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_external_scroll_input_seq = 11; + session.scroll_capture.inflight_request_id = Some(41); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: 11, + input_direction: Some(ScrollDirection::Down), + }); + + let scroll_session_before = + format!("{:?}", session.scroll_capture.session.as_ref().unwrap()); + + session.handle_missing_scroll_region(monitor, capture_rect, 41); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!( + format!("{:?}", session.scroll_capture.session.as_ref().unwrap()), + scroll_session_before + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_stays_on_stream_path_without_worker_fallback() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + + session.maybe_tick_scroll_capture(); + + assert!(!session.scroll_capture.paused); + assert!(session.state.error_message.is_none()); + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert!(matches!( + session.scroll_capture.live_stream.as_ref().unwrap().debug_last_request_kind(), + Some("ordered_rgba_regions_after_seq_nonblocking") + | Some("refresh_monitor_nonblocking_if_stale") + )); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_drains_external_input_without_a_new_stream_frame() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let rect = RectPoints::new(100, 120, 200, 240); + let tick_at = Instant::now(); + let event_at = tick_at - Duration::from_millis(1); + let events = Arc::new([(1, event_at, 150.0, 160.0, -4.0, true, false)]); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); + + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + + session.maybe_tick_scroll_capture(); + + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert!(session.scroll_capture.input_gesture_active); + assert_eq!(session.scroll_capture.last_external_scroll_input_seq, 1); + assert!(session.state.error_message.is_none()); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_does_not_synthesize_preview_growth_from_input_without_semantic_sample() + { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + ]; + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let rect = RectPoints::new(100, 120, 200, 240); + let tick_at = Instant::now(); + let event_at = tick_at - Duration::from_millis(1); + let events = Arc::new([(1, event_at, 150.0, 160.0, -4.0, true, false)]); + let base_frame = make_scroll_capture_window(&document, 3, 0, 5); + let latest_frame = make_scroll_capture_window(&document, 3, 1, 5); + let mut session = OverlaySession::new(); + let scroll_session = ScrollSession::new(base_frame.clone(), 320).unwrap(); + let committed_preview = scroll_session.preview_image().clone(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.scroll_capture.session = Some(scroll_session); + session.scroll_capture.preview_committed_image = Some(committed_preview.clone()); + session.scroll_capture.preview_display_image = Some(committed_preview.clone()); + session.scroll_capture.preview_latest_frame = Some(latest_frame); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); + + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + + session.maybe_tick_scroll_capture(); + + assert_eq!(session.scroll_capture.preview_display_image.as_ref(), Some(&committed_preview)); + assert_eq!(scroll_capture_export_height(&session), base_frame.height()); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_does_not_double_count_preview_growth_from_same_latest_frame() { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + ]; + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let rect = RectPoints::new(100, 120, 200, 240); + let tick_at = Instant::now(); + let event_at = tick_at - Duration::from_millis(1); + let events = Arc::new([(1, event_at, 150.0, 160.0, -4.0, true, false)]); + let base_frame = make_scroll_capture_window(&document, 3, 0, 5); + let moved_frame = make_scroll_capture_window(&document, 3, 1, 5); + let mut session = OverlaySession::new(); + let mut scroll_session = ScrollSession::new(base_frame, 320).unwrap(); + assert!(matches!( + scroll_session.observe_downward_sample(moved_frame.clone()).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + )); + let committed_preview = scroll_session.preview_image().clone(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.scroll_capture.session = Some(scroll_session); + session.scroll_capture.preview_committed_image = Some(committed_preview.clone()); + session.scroll_capture.preview_display_image = Some(committed_preview.clone()); + session.scroll_capture.preview_latest_frame = Some(moved_frame); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); + + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + + session.maybe_tick_scroll_capture(); + + assert_eq!(session.scroll_capture.preview_display_image.as_ref(), Some(&committed_preview)); + assert_eq!(scroll_capture_export_height(&session), committed_preview.height()); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_recovers_after_blocked_overshot_frame() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_browser_like_worker_capture_window(512, 640, 0); + let blocked = make_browser_like_worker_capture_window(512, 640, 760); + let followup = make_browser_like_worker_capture_window(512, 640, 844); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([Some(blocked), Some(followup)])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 640); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 0); + + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 724); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_retries_immediately_after_blocked_overshot_frame_during_fresh_downward_input() + { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_browser_like_worker_capture_window(512, 640, 0); + let blocked = make_browser_like_worker_capture_window(512, 640, 760); + let followup = make_browser_like_worker_capture_window(512, 640, 844); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([Some(blocked), Some(followup)])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 640); + + session.scroll_capture.last_external_scroll_input_seq = 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.maybe_tick_scroll_capture(); + + assert!( + session.scroll_capture.inflight_request_id.is_some(), + "fresh downward input after a blocked worker frame should retry immediately" + ); + + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + assert_eq!(scroll_capture_export_height(&session), 724); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_recovers_across_interleaved_no_frame_and_blocked_browser_steps() + { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([ + None, + Some(make_browser_like_worker_capture_window(512, 640, 84)), + Some(make_browser_like_worker_capture_window(512, 640, 700)), + Some(make_browser_like_worker_capture_window(512, 640, 784)), + None, + Some(make_browser_like_worker_capture_window(512, 640, 868)), + ])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some( + ScrollSession::new(make_browser_like_worker_capture_window(512, 640, 0), 320).unwrap(), + ); + enable_test_worker_scroll_capture_path(&mut session); + + for expected_top_y in [84_i32, 168, 252] { + let mut attempts = 0_u8; + while session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y() + < expected_top_y + { + attempts = attempts.saturating_add(1); + assert!( + attempts <= 4, + "worker path failed to recover to expected_top_y={expected_top_y}" + ); + + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = + session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); + session.scroll_capture.next_sample_at = + Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + + session.scroll_capture.last_external_scroll_input_seq = + session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + drain_scroll_capture_worker_until_idle(&mut session); + } + + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), + expected_top_y + ); + assert_eq!(scroll_capture_export_height(&session), 640 + expected_top_y as u32); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_keeps_same_direction_superseded_response() { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_sparse_worker_capture_window(512, 640, 0); + let moved = make_sparse_worker_capture_window(512, 640, 180); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([Some(moved)])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + session.scroll_capture.last_external_scroll_input_seq = 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + drain_scroll_capture_worker_until_idle(&mut session); + + assert_eq!(scroll_capture_export_height(&session), 820); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 180); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_commits_successive_browser_like_frames_after_newer_same_direction_input() + { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([ + Some(make_browser_like_worker_capture_window(512, 640, 84)), + Some(make_browser_like_worker_capture_window(512, 640, 168)), + Some(make_browser_like_worker_capture_window(512, 640, 252)), + ])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some( + ScrollSession::new(make_browser_like_worker_capture_window(512, 640, 0), 320).unwrap(), + ); + enable_test_worker_scroll_capture_path(&mut session); + + for (step, expected_top_y) in [84_i32, 168, 252].into_iter().enumerate() { + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + + session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + drain_scroll_capture_worker_until_idle(&mut session); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), + expected_top_y + ); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + 640 + expected_top_y as u32 + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_drops_opposite_direction_superseded_response() { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_sparse_worker_capture_window(512, 640, 0); + let moved = make_sparse_worker_capture_window(512, 640, 180); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([Some(moved)])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + session.scroll_capture.last_external_scroll_input_seq = 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Up); + drain_scroll_capture_worker_until_idle(&mut session); + + assert_eq!(scroll_capture_export_height(&session), 640); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 0); + } + + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_retries_immediately_after_no_new_frame_during_fresh_downward_input() + { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_browser_like_worker_capture_window(512, 640, 0); + let moved = make_browser_like_worker_capture_window(512, 640, 84); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([None, Some(moved)])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(scroll_capture_export_height(&session), 640); + + session.scroll_capture.last_external_scroll_input_seq = 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.maybe_tick_scroll_capture(); + + assert!( + session.scroll_capture.inflight_request_id.is_some(), + "fresh downward input after a worker no-frame response should retry immediately" + ); + + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + assert_eq!(scroll_capture_export_height(&session), 724); + } + + #[cfg(target_os = "macos")] + #[test] + fn handle_scroll_input_ready_drains_input_and_polls_stream_fallback() { + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let rect = RectPoints::new(100, 120, 200, 240); + let handled_at = Instant::now(); + let event_at = handled_at - Duration::from_millis(1); + let events = Arc::new([(1, event_at, 150.0, 160.0, -4.0, true, false)]); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); + + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + + assert!(matches!(session.handle_scroll_input_ready(), OverlayControl::Continue)); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert!(session.scroll_capture.input_gesture_active); + assert_eq!(session.scroll_capture.last_external_scroll_input_seq, 1); + assert!(matches!( + session.scroll_capture.live_stream.as_ref().unwrap().debug_last_request_kind(), + Some("ordered_rgba_regions_after_seq_nonblocking") + | Some("refresh_monitor_nonblocking_if_stale") + )); + } + + #[cfg(target_os = "macos")] + #[test] + fn drain_external_scroll_input_worker_path_does_not_arm_live_stream_stale_grace() { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let through = Instant::now(); + let recorded_at = through - Duration::from_millis(1); + let events = Arc::new([(1, recorded_at, 150.0, 160.0, -4.0, false, false)]); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + enable_test_worker_scroll_capture_path(&mut session); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); + + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + + session.drain_external_scroll_input_events_through(through); + + assert_eq!(session.scroll_capture.last_external_scroll_input_seq, 1); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert!(!session.scroll_capture.input_gesture_active); + assert_eq!(session.scroll_capture.live_stream_stale_grace, None); + } + + #[cfg(target_os = "macos")] + #[test] + fn force_stream_refresh_stays_disabled_while_downward_gesture_is_still_active() { + let mut session = OverlaySession::new(); + let now = Instant::now(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_direction_at = Some(now); session.scroll_capture.input_gesture_active = true; + session.scroll_capture.downward_motion_rows_pending = 512.0; - assert_eq!( - session - .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 3, 5)) - .transpose() - .unwrap(), - Some(ScrollObserveOutcome::Committed { - direction: ScrollDirection::Down, - growth_rows: 1, - }) + assert!(!session.scroll_capture_should_force_stream_refresh_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn stale_stream_refresh_stays_disabled_while_gesture_is_still_active() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_stream_event_at = Some( + now - SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + + Duration::from_millis(1), ); + + assert!(!session.scroll_capture_should_schedule_stale_stream_refresh_at(now)); } #[cfg(target_os = "macos")] #[test] - fn missing_worker_scroll_frame_clears_inflight_without_mutating_session() { + fn stale_stream_refresh_reenables_after_gesture_ends() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_gesture_active = false; + + assert!(session.scroll_capture_should_schedule_stale_stream_refresh_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn stale_stream_refresh_reenables_during_gesture_after_stream_goes_dead() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.last_stream_event_at = Some( + now - SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + - Duration::from_millis(1), + ); + + assert!(session.scroll_capture_should_schedule_stale_stream_refresh_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn post_stall_burst_search_stays_enabled_during_active_gesture_when_downward_backlog_is_fresh() + { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(now); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(session.scroll_capture_should_allow_post_stall_burst_search_at(81, now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn force_stream_refresh_stays_enabled_for_fresh_pending_downward_motion_after_gesture_end() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = + Some(now - SCROLL_CAPTURE_INPUT_FRESHNESS + Duration::from_millis(50)); + session.scroll_capture.input_gesture_active = false; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(session.scroll_capture_should_force_stream_refresh_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn force_stream_refresh_stops_after_downward_input_becomes_stale() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = + Some(now - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(1)); + session.scroll_capture.input_gesture_active = false; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(!session.scroll_capture_should_force_stream_refresh_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn post_stall_burst_search_stays_enabled_while_fresh_downward_backlog_remains() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(now); + session.scroll_capture.input_gesture_active = false; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(session.scroll_capture_should_allow_post_stall_burst_search_at(81, now)); + assert!(session.scroll_capture_should_allow_post_stall_burst_search_at( + 82, + now + Duration::from_millis(50) + )); + } + + #[cfg(target_os = "macos")] + #[test] + fn post_stall_burst_search_arms_for_large_capture_time_gap_even_when_frame_seq_is_contiguous() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(now); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.downward_motion_rows_pending = 512.0; + session.scroll_capture.last_consumed_stream_frame_captured_at = Some( + now - SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + - Duration::from_millis(1), + ); + + assert!(session.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(now)); + } + + #[cfg(target_os = "macos")] + #[test] + fn consuming_live_frame_backlog_arms_time_gap_burst_after_draining_fresh_input() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -15903,8 +17815,6 @@ mod tests { [40, 0, 0, 255], [50, 0, 0, 255], [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], ]; let monitor = MonitorRect { id: 1, @@ -15913,61 +17823,78 @@ mod tests { height: 800, scale_factor_x1000: 1_000, }; - let capture_rect = RectPoints::new(100, 120, 200, 240); + let rect = RectPoints::new(100, 120, 200, 240); + let captured_at = Instant::now(); + let event_at = captured_at - Duration::from_millis(1); + let events = Arc::new([(1, event_at, 150.0, 160.0, -74.0, true, false)]); let mut session = OverlaySession::new(); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(monitor); - session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); - session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = Some(Instant::now()); - session.scroll_capture.input_gesture_active = true; - session.scroll_capture.last_external_scroll_input_seq = 11; - session.scroll_capture.inflight_request_id = Some(41); - session.scroll_capture.inflight_request_observation = - Some(InflightScrollCaptureObservation { - input_direction: Some(ScrollDirection::Down), - was_observable: true, - external_input_seq: 11, - }); + session.scroll_capture.last_consumed_stream_frame_captured_at = Some( + captured_at + - SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + - Duration::from_millis(1), + ); + session.set_external_scroll_input_drain_reader(Arc::new({ + let events = Arc::clone(&events); - let scroll_session_before = - format!("{:?}", session.scroll_capture.session.as_ref().unwrap()); + move |after_seq, through| { + events + .iter() + .copied() + .filter(|event| event.0 > after_seq && event.1 <= through) + .collect() + } + })); + session.test_push_scroll_capture_live_frame(ScrollCaptureLiveFrame { + frame_seq: 9, + captured_at, + image: make_scroll_capture_window(&document, 3, 0, 5), + }); - session.handle_missing_scroll_region(monitor, capture_rect, 41); + session.test_consume_scroll_capture_backlog(1); - assert_eq!(session.scroll_capture.inflight_request_id, None); - assert_eq!(session.scroll_capture.inflight_request_observation, None); - assert_eq!( - format!("{:?}", session.scroll_capture.session.as_ref().unwrap()), - scroll_session_before - ); + assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); + assert_eq!(session.scroll_capture.last_external_scroll_input_seq, 1); + assert_eq!(session.scroll_capture.pending_post_stall_burst_after_seq, Some(8)); } #[cfg(target_os = "macos")] #[test] - fn maybe_tick_scroll_capture_stays_on_stream_path_without_worker_fallback() { - let monitor = MonitorRect { - id: 1, - origin: GlobalPoint::new(0, 0), - width: 1_000, - height: 800, - scale_factor_x1000: 1_000, - }; + fn post_stall_burst_search_does_not_arm_for_small_capture_time_gap() { let mut session = OverlaySession::new(); + let now = Instant::now(); - session.scroll_capture.active = true; - session.scroll_capture.monitor = Some(monitor); - session.scroll_capture.capture_rect_pixels = Some(RectPoints::new(100, 120, 200, 240)); - session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(now); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.downward_motion_rows_pending = 512.0; + session.scroll_capture.last_consumed_stream_frame_captured_at = Some( + now - SCROLL_CAPTURE_ACTIVE_GESTURE_STALE_REFRESH_DEAD_WINDOW + + Duration::from_millis(10), + ); - session.maybe_tick_scroll_capture(); + assert!(!session.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(now)); + } - assert!(!session.scroll_capture.paused); - assert!(session.state.error_message.is_none()); - assert_eq!(session.scroll_capture.inflight_request_id, None); + #[cfg(target_os = "macos")] + #[test] + fn post_stall_burst_search_stops_after_downward_backlog_goes_stale() { + let mut session = OverlaySession::new(); + let now = Instant::now(); + + session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = + Some(now - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(1)); + session.scroll_capture.input_gesture_active = false; + session.scroll_capture.downward_motion_rows_pending = 512.0; + + assert!(!session.scroll_capture_should_allow_post_stall_burst_search_at(81, now)); } #[cfg(target_os = "macos")] @@ -16205,7 +18132,7 @@ mod tests { } #[test] - fn upward_input_with_lower_frame_never_appends_growth() { + fn downward_frame_motion_commits_even_with_legacy_upward_input_direction() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -16242,33 +18169,6 @@ mod tests { session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; - assert!(matches!( - session - .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 2, 5)) - .transpose() - .unwrap(), - Some( - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange - ) - )); - assert_eq!( - session.scroll_capture.session.as_ref().unwrap().export_image().height(), - height_after_first_append - ); - assert!(matches!( - session - .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 2, 5)) - .transpose() - .unwrap(), - Some(ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange) - )); - - session.scroll_capture.input_direction = Some(ScrollDirection::Down); - session.scroll_capture.input_direction_at = Some(Instant::now()); - session.scroll_capture.input_gesture_active = true; - assert_eq!( session .observe_scroll_capture_frame(make_scroll_capture_window(&document, 3, 2, 5)) @@ -16279,6 +18179,11 @@ mod tests { growth_rows: 1, }) ); + + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + height_after_first_append + 1 + ); } #[cfg(target_os = "macos")] @@ -16343,6 +18248,12 @@ mod tests { assert!(!FrozenToolbarTool::Save.is_mode_tool()); } + #[test] + fn frozen_toolbar_scroll_tool_uses_scroll_specific_iconography() { + assert_eq!(FrozenToolbarTool::Scroll.label(), "Scroll Capture"); + assert_eq!(FrozenToolbarTool::Scroll.icon(), regular::MOUSE_SCROLL); + } + #[test] fn frozen_toolbar_export_tools_require_final_capture() { assert!(!FrozenToolbarTool::Pointer.requires_final_capture()); diff --git a/packages/rsnap-overlay/src/overlay/replay_support.rs b/packages/rsnap-overlay/src/overlay/replay_support.rs new file mode 100644 index 00000000..7785d402 --- /dev/null +++ b/packages/rsnap-overlay/src/overlay/replay_support.rs @@ -0,0 +1,1517 @@ +use std::{ + path::Path, + time::{Duration, Instant}, +}; + +use color_eyre::eyre::{Result, WrapErr, eyre}; +use image::RgbaImage; +use serde::Serialize; + +use super::{ + GlobalPoint, MonitorRect, OverlaySession, RectPoints, ScrollCaptureFrameSource, + ScrollDirection, ScrollObserveOutcome, ScrollSession, + trace_recording::{ + LoadedScrollCaptureLiveTrace, ScrollCaptureLiveTraceEntry, ScrollCaptureTraceFrameSource, + ScrollCaptureTraceRecordedOutcome, + }, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +/// Public replay outcome surface that does not leak private scroll-capture internals. +pub enum ScrollCaptureReplayOutcome { + /// The step did not change preview or export state. + NoChange, + /// The step updated preview state without committing stitched growth. + PreviewUpdated, + /// The step detected upward or rewind-like motion and failed closed. + UnsupportedUp, + /// The step committed downward stitched growth. + CommittedDown { + /// Number of newly proven rows appended during this step. + growth_rows: u32, + }, +} + +impl From for ScrollCaptureReplayOutcome { + fn from(value: ScrollObserveOutcome) -> Self { + match value { + ScrollObserveOutcome::NoChange => Self::NoChange, + ScrollObserveOutcome::PreviewUpdated => Self::PreviewUpdated, + ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } => { + Self::UnsupportedUp + }, + ScrollObserveOutcome::UnsupportedDirection { .. } => Self::NoChange, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows } => { + Self::CommittedDown { growth_rows } + }, + ScrollObserveOutcome::Committed { .. } => Self::NoChange, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +/// Strategy used when replaying recorded frames back through scroll-capture logic. +pub enum RecordedScrollCaptureReplayMode { + /// Reuse the recorded frame source for each step. + RecordedSource, + /// Force every recorded frame through the macOS worker pairwise path. + ForceWorkerPairwise, +} + +fn classify_replayed_outcome( + outcome: ScrollObserveOutcome, + previous_replayed_export_height: Option, + replayed_export_height: u32, + previous_replayed_preview_height: Option, + replayed_preview_height: u32, +) -> ScrollCaptureReplayOutcome { + let replayed_outcome: ScrollCaptureReplayOutcome = outcome.into(); + + if replayed_outcome == ScrollCaptureReplayOutcome::NoChange + && previous_replayed_export_height == Some(replayed_export_height) + && previous_replayed_preview_height + .is_some_and(|previous| replayed_preview_height > previous) + { + ScrollCaptureReplayOutcome::PreviewUpdated + } else { + replayed_outcome + } +} + +#[derive(Clone, Debug, Serialize)] +/// Deterministic replay summary for one recorded live trace. +pub struct RecordedScrollCaptureReplaySummary { + /// Replay strategy used for this summary. + pub replay_mode: RecordedScrollCaptureReplayMode, + /// Stable trace id from the manifest. + pub trace_id: String, + /// Manifest path used to load the trace. + pub manifest_path: std::path::PathBuf, + /// Final stitched export height after replaying the recorded trace. + pub final_export_height: u32, + /// Final preview height after replaying the recorded trace. + pub final_preview_height: u32, + /// Final viewport top ledger after replaying the recorded trace. + pub final_viewport_top_y: i32, + /// Final stitched export height recorded during the live session, when present. + pub recorded_final_export_height: Option, + /// Final preview height recorded during the live session, when present. + pub recorded_final_preview_height: Option, + /// Final recorded live-trace preview artifact, when present. + pub final_preview_path: Option, + /// Final recorded live-trace export artifact, when present. + pub final_export_path: Option, + /// First frame where recorded and replayed outcomes diverged. + pub first_outcome_divergence_frame: Option, + /// First frame where replayed export height drifted from the recorded live trace. + pub first_export_height_drift_frame: Option, + /// First frame where replayed preview height drifted from the recorded live trace. + pub first_preview_height_drift_frame: Option, + /// Largest committed growth recorded during the live session. + pub max_recorded_committed_growth_rows: u32, + /// Largest committed growth observed while replaying the live trace. + pub max_replayed_committed_growth_rows: u32, + /// Largest step-to-step export-height jump recorded during the live session. + pub max_recorded_export_jump: u32, + /// Largest step-to-step preview-height jump recorded during the live session. + pub max_recorded_preview_jump: u32, + /// Largest step-to-step export-height jump observed while replaying the live trace. + pub max_replayed_export_jump: u32, + /// Largest step-to-step preview-height jump observed while replaying the live trace. + pub max_replayed_preview_jump: u32, + /// First frame where semantic analysis flagged a likely incorrect recorded behavior. + pub first_semantic_issue_frame: Option, + /// First frame where visible downward motion was recorded but no stitched growth occurred. + pub first_missed_downward_motion_frame: Option, + /// First frame where committed growth consumed materially less motion than visible in the recorded frames. + pub first_underconsumed_downward_motion_frame: Option, + /// First frame where committed growth exceeded the recorded frame-to-frame shift estimate by a large margin. + pub first_growth_overshoot_frame: Option, + /// Ordered per-frame results observed during replay. + pub step_results: Vec, +} + +#[derive(Clone, Debug, Serialize)] +/// One deterministic replay step result for a recorded live trace. +pub struct RecordedScrollCaptureReplayStepResult { + /// Zero-based frame index within the trace. + pub frame_index: usize, + /// Relative frame image path inside the trace manifest. + pub frame_path: String, + /// Milliseconds since the trace start when this frame was observed live. + pub observed_at_ms: u64, + /// Public frame-source surface for the recorded frame. + pub frame_source: RecordedScrollCaptureReplayFrameSource, + /// Gap from the previous live-stream frame sequence, when the source was SCStream-backed. + pub live_frame_gap: Option, + /// Outcome recorded during the live session. + pub recorded_outcome: RecordedScrollCaptureReplayRecordedOutcome, + /// Outcome observed while replaying the recorded trace offline. + pub replayed_outcome: ScrollCaptureReplayOutcome, + /// Export height after the replay step completes. + pub export_height: u32, + /// Preview height after the replay step completes. + pub preview_height: u32, + /// Session-side preview-display height after the replay step completes. + pub session_preview_height: u32, + /// Export height recorded during the live session after this frame, when present. + pub recorded_export_height: Option, + /// Preview height recorded during the live session after this frame, when present. + pub recorded_preview_height: Option, + /// Viewport top ledger after the replay step completes. + pub viewport_top_y: i32, + /// Last commit decision source visible after this frame replay completes. + pub last_commit_decision_source: Option<&'static str>, + /// Last commit detected motion rows visible after this frame replay completes. + pub last_commit_detected_motion_rows: Option, + /// Last fail-closed block reason visible after this frame replay completes. + pub last_block_reason: Option<&'static str>, + /// Replay-side downward sample registration result observed for this frame. + pub replayed_downward_sample_registration_result: Option<&'static str>, + /// Replay-side downward sample registration source observed for this frame. + pub replayed_downward_sample_registration_source: Option<&'static str>, + /// Replay-side downward sample registration motion rows observed for this frame. + pub replayed_downward_sample_registration_motion_rows: Option, + /// Replay-side provisional viewport top inferred from the registration source, when any. + pub replayed_downward_sample_registration_provisional_viewport_top_y: Option, + /// Replay-side observed-sample registration result before source arbitration. + pub replayed_observed_sample_registration_result: Option<&'static str>, + /// Replay-side observed-sample registration no-match reason before source arbitration. + pub replayed_observed_sample_registration_reason: Option<&'static str>, + /// Replay-side observed-sample registration motion rows before source arbitration. + pub replayed_observed_sample_registration_motion_rows: Option, + /// Replay-side observed-sample registration mean diff before source arbitration. + pub replayed_observed_sample_registration_mean_abs_diff_x100: Option, + /// Replay-side preview-local registration result before source arbitration. + pub replayed_preview_only_local_registration_result: Option<&'static str>, + /// Replay-side preview-local registration no-match reason before source arbitration. + pub replayed_preview_only_local_registration_reason: Option<&'static str>, + /// Replay-side preview-local registration motion rows before source arbitration. + pub replayed_preview_only_local_registration_motion_rows: Option, + /// Replay-side preview-local registration mean diff before source arbitration. + pub replayed_preview_only_local_registration_mean_abs_diff_x100: Option, + /// Candidate count that reached viewport selection during replay for this frame, when observed. + pub replayed_downward_viewport_candidate_count: Option, + /// Candidate set before committed/local pruning during replay for this frame, when observed. + pub replayed_downward_viewport_candidates_before_prune: Option, + /// Candidate set after committed/local pruning during replay for this frame, when observed. + pub replayed_downward_viewport_candidates_after_prune: Option, + /// Last local continuity hint visible while replaying this frame. + pub replayed_sample_eval_last_motion_rows_hint: Option, + /// Last transient motion hint visible while replaying this frame. + pub replayed_sample_eval_transient_motion_rows_hint: Option, + /// Effective motion hint visible while replaying this frame. + pub replayed_sample_eval_effective_motion_rows_hint: Option, + /// Whether burst search remained enabled while replaying this frame. + pub replayed_sample_eval_transient_burst_search_enabled: bool, + /// Preview-only local viewport ledger still retained after this frame, when any. + pub replayed_preview_only_local_viewport_top_y: Option, + /// Replay-side pending downward input rows visible after this frame. + pub replayed_downward_motion_rows_pending: f64, + /// Replay-side gesture-active flag visible after this frame. + pub replayed_input_gesture_active: bool, + /// Session-side preview display mode selected after this frame. + pub replayed_session_preview_display_mode: &'static str, + /// Session-side hinted preview motion hint, when the hinted path is available. + pub replayed_session_preview_hinted_motion_rows_hint: Option, + /// Session-side hinted preview frame source, when the hinted path is available. + pub replayed_session_preview_hinted_frame_source: Option<&'static str>, + /// Overlay-side pending-derived motion hint visible after this frame. + pub replayed_overlay_preview_motion_rows_hint: Option, + /// Overlay-side provisional motion hint visible after this frame. + pub replayed_overlay_preview_provisional_motion_rows_hint: Option, + /// Existing overlay-preview image candidate considered during refresh, when any. + pub replayed_overlay_preview_existing_candidate_height: Option, + /// Existing overlay-preview image candidate motion hint considered during refresh, when any. + pub replayed_overlay_preview_existing_candidate_motion_rows_hint: Option, + /// Retained overlay-preview ledger candidate considered during refresh, when any. + pub replayed_overlay_preview_ledger_candidate_height: Option, + /// Retained overlay-preview ledger motion hint considered during refresh, when any. + pub replayed_overlay_preview_ledger_candidate_motion_rows_hint: Option, + /// Overlay-side retained candidate height considered during refresh, when any. + pub replayed_overlay_preview_retained_candidate_height: Option, + /// Overlay-side retained candidate motion hint considered during refresh, when any. + pub replayed_overlay_preview_retained_candidate_motion_rows_hint: Option, + /// Whether the retained ledger hint matched the current pending motion band. + pub replayed_overlay_preview_retained_hint_matches_motion_rows: bool, + /// Whether the overlay refresh considered the fresh-latest-frame authority path. + pub replayed_overlay_preview_fresh_latest_frame_can_drive: bool, + /// Retained overlay-preview ledger height visible after this frame, when any. + pub replayed_retained_overlay_preview_height: Option, + /// Retained overlay-preview ledger motion hint visible after this frame, when any. + pub replayed_retained_overlay_preview_motion_rows_hint: Option, + /// Whether overlay refresh saw a strong unresolved registration. + pub replayed_overlay_preview_strong_unresolved_registration: bool, + /// Whether overlay refresh had a latest frame to work from. + pub replayed_overlay_preview_latest_frame_present: bool, + /// Whether overlay refresh selected any non-session overlay preview. + pub replayed_overlay_preview_used_provisional: bool, + /// Estimated downward shift visible between the previous recorded frame and this recorded frame. + pub recorded_estimated_downward_shift_rows: Option, + /// Semantic issue detected directly from the recorded frame progression, when any. + pub semantic_issue: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Semantic failure classes inferred from recorded frame-to-frame motion. +pub enum RecordedScrollCaptureSemanticIssue { + /// Recorded frames moved downward but the recorded outcome did not convert that into growth. + MissedDownwardMotion, + /// Recorded frames moved downward significantly more than the recorded committed growth. + UnderconsumedDownwardMotion, + /// Recorded committed growth exceeded the visible recorded frame-to-frame shift by a large margin. + GrowthExceedsRecordedShift, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Public frame-source surface for one replayed live-trace step. +pub enum RecordedScrollCaptureReplayFrameSource { + Worker { request_id: u64 }, + LiveStream { frame_seq: u64 }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Public recorded-outcome surface for one frame in a replayed live trace. +pub enum RecordedScrollCaptureReplayRecordedOutcome { + /// The live frame did not change preview or export state. + NoChange, + /// The live frame updated preview state without committing growth. + PreviewUpdated, + /// The live frame detected unsupported motion. + Unsupported { + /// Direction recorded during the live session. + direction: &'static str, + }, + /// The live frame committed stitched growth. + Committed { + /// Direction recorded during the live session. + direction: &'static str, + /// Newly appended rows recorded during the live session. + growth_rows: u32, + }, + /// The live frame recorded an observation error. + Error { + /// Error string captured during the live session. + message: String, + }, +} + +impl From for RecordedScrollCaptureReplayRecordedOutcome { + fn from(value: ScrollCaptureTraceRecordedOutcome) -> Self { + match value { + ScrollCaptureTraceRecordedOutcome::NoChange => Self::NoChange, + ScrollCaptureTraceRecordedOutcome::PreviewUpdated => Self::PreviewUpdated, + ScrollCaptureTraceRecordedOutcome::UnsupportedDirection { direction } => { + Self::Unsupported { + direction: match direction { + super::trace_recording::ScrollCaptureTraceDirection::Up => "up", + super::trace_recording::ScrollCaptureTraceDirection::Down => "down", + }, + } + }, + ScrollCaptureTraceRecordedOutcome::Committed { direction, growth_rows } => { + Self::Committed { + direction: match direction { + super::trace_recording::ScrollCaptureTraceDirection::Up => "up", + super::trace_recording::ScrollCaptureTraceDirection::Down => "down", + }, + growth_rows, + } + }, + ScrollCaptureTraceRecordedOutcome::Error { message } => Self::Error { message }, + } + } +} + +impl From + for RecordedScrollCaptureReplayFrameSource +{ + fn from(value: super::trace_recording::ScrollCaptureTraceFrameSource) -> Self { + match value { + super::trace_recording::ScrollCaptureTraceFrameSource::Worker { request_id } => { + Self::Worker { request_id } + }, + super::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { + Self::LiveStream { frame_seq } + }, + } + } +} + +/// Replays one recorded live trace through shipping overlay logic. +pub fn replay_recorded_scroll_capture_trace( + manifest_path: impl AsRef, +) -> Result { + replay_recorded_scroll_capture_trace_with_mode( + manifest_path, + RecordedScrollCaptureReplayMode::RecordedSource, + ) +} + +/// Replays one recorded live trace through shipping overlay logic with an explicit frame-source mode. +pub fn replay_recorded_scroll_capture_trace_with_mode( + manifest_path: impl AsRef, + replay_mode: RecordedScrollCaptureReplayMode, +) -> Result { + let trace = LoadedScrollCaptureLiveTrace::load(manifest_path)?; + let (mut session, started_at) = initialize_replay_session(&trace)?; + let replay_stats = replay_trace_entries(&trace, &mut session, started_at, replay_mode)?; + + finalize_replay_summary(trace, &session, replay_stats, replay_mode) +} + +#[derive(Default)] +struct ReplayStats { + step_results: Vec, + previous_recorded_export_height: Option, + previous_recorded_preview_height: Option, + previous_replayed_export_height: Option, + previous_replayed_preview_height: Option, + previous_live_frame_seq: Option, + previous_recorded_frame: Option, + max_recorded_export_jump: u32, + max_recorded_preview_jump: u32, + max_replayed_export_jump: u32, + max_replayed_preview_jump: u32, + max_recorded_committed_growth_rows: u32, + max_replayed_committed_growth_rows: u32, +} + +fn initialize_replay_session( + trace: &LoadedScrollCaptureLiveTrace, +) -> Result<(OverlaySession, Instant)> { + let mut session = OverlaySession::new(); + let started_at = Instant::now(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(replay_monitor_from_trace(trace)); + session.scroll_capture.capture_rect_pixels = Some(replay_capture_rect_from_trace(trace)); + session.scroll_capture.session = + Some(ScrollSession::new(trace.base_frame.clone(), trace.manifest.preview_width_px)?); + session.refresh_scroll_preview_committed_image(); + session.scroll_capture.preview_latest_frame = Some(trace.base_frame.clone()); + session.refresh_scroll_preview_display_image(); + + Ok((session, started_at)) +} + +fn replay_trace_entries( + trace: &LoadedScrollCaptureLiveTrace, + session: &mut OverlaySession, + started_at: Instant, + replay_mode: RecordedScrollCaptureReplayMode, +) -> Result { + let mut replay_stats = ReplayStats::default(); + + for entry in &trace.manifest.entries { + match entry { + ScrollCaptureLiveTraceEntry::Input(input) => { + apply_replayed_input(session, input, started_at); + }, + ScrollCaptureLiveTraceEntry::Frame(frame) => { + replay_frame_entry( + trace, + session, + frame, + started_at, + replay_mode, + &mut replay_stats, + )?; + }, + } + } + + Ok(replay_stats) +} + +fn apply_replayed_input( + session: &mut OverlaySession, + input: &super::trace_recording::ScrollCaptureTraceInputEntry, + started_at: Instant, +) { + session.apply_external_scroll_input_delta_y( + input.cursor_global_x, + input.cursor_global_y, + input.delta_y, + input.gesture_active, + input.gesture_ended, + started_at + Duration::from_millis(input.applied_at_ms), + ); + session.refresh_scroll_preview_display_image(); +} + +fn replay_frame_entry( + trace: &LoadedScrollCaptureLiveTrace, + session: &mut OverlaySession, + frame: &super::trace_recording::ScrollCaptureTraceFrameEntry, + started_at: Instant, + replay_mode: RecordedScrollCaptureReplayMode, + replay_stats: &mut ReplayStats, +) -> Result<()> { + let recorded_export_height = + frame.snapshot_after.export_dimensions.map(|dimensions| dimensions[1]); + let recorded_preview_height = + frame.snapshot_after.preview_dimensions.map(|dimensions| dimensions[1]); + update_recorded_height_jumps(replay_stats, recorded_export_height, recorded_preview_height); + + let image = image::open(trace.resolve_frame_path(&frame.frame_path)) + .wrap_err("failed to open recorded live trace frame")? + .into_rgba8(); + let recorded_estimated_downward_shift_rows = replay_stats + .previous_recorded_frame + .as_ref() + .and_then(|previous| estimate_recorded_downward_shift_rows(previous, &image)); + let observed_at = started_at + Duration::from_millis(frame.observed_at_ms); + let outcome = match replay_mode { + RecordedScrollCaptureReplayMode::RecordedSource => match frame.frame_source { + ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => session + .replay_recorded_live_stream_frame( + image.clone(), + frame_seq, + observed_at, + frame.allow_stale_input, + ), + ScrollCaptureTraceFrameSource::Worker { .. } => session.handle_scroll_capture_frame( + image.clone(), + replay_frame_source(frame.frame_source), + frame.allow_stale_input, + observed_at, + ), + }, + RecordedScrollCaptureReplayMode::ForceWorkerPairwise => session + .handle_scroll_capture_frame( + image.clone(), + ScrollCaptureFrameSource::Worker { + request_id: replay_stats.step_results.len() as u64, + }, + frame.allow_stale_input, + observed_at, + ), + } + .transpose()? + .ok_or_else(|| { + eyre!( + "recorded trace frame {} did not observe because the session vanished", + frame.frame_path + ) + })?; + let active_session = session.scroll_capture.session.as_ref().ok_or_else(|| { + eyre!("scroll-capture session missing after replaying recorded frame {}", frame.frame_path) + })?; + let telemetry = active_session.commit_telemetry(); + let frame_source: RecordedScrollCaptureReplayFrameSource = frame.frame_source.into(); + let live_frame_gap = update_live_frame_gap(replay_stats, frame_source.clone()); + let recorded_outcome: RecordedScrollCaptureReplayRecordedOutcome = frame.outcome.clone().into(); + let replayed_export_height = active_session.export_image().height(); + let replayed_session_preview_height = active_session.preview_display_image().height(); + let replayed_preview_height = session + .scroll_capture_preview_dimensions() + .map_or(replayed_session_preview_height, |dimensions| dimensions[1]); + let replayed_outcome = classify_replayed_outcome( + outcome, + replay_stats.previous_replayed_export_height, + replayed_export_height, + replay_stats.previous_replayed_preview_height, + replayed_preview_height, + ); + let semantic_issue = + classify_recorded_semantic_issue(&recorded_outcome, recorded_estimated_downward_shift_rows); + + if let RecordedScrollCaptureReplayRecordedOutcome::Committed { growth_rows, .. } = + recorded_outcome + { + replay_stats.max_recorded_committed_growth_rows = + replay_stats.max_recorded_committed_growth_rows.max(growth_rows); + } + if let ScrollCaptureReplayOutcome::CommittedDown { growth_rows } = replayed_outcome { + replay_stats.max_replayed_committed_growth_rows = + replay_stats.max_replayed_committed_growth_rows.max(growth_rows); + } + update_replayed_height_jumps(replay_stats, replayed_export_height, replayed_preview_height); + + replay_stats.step_results.push(RecordedScrollCaptureReplayStepResult { + frame_index: replay_stats.step_results.len(), + frame_path: frame.frame_path.clone(), + observed_at_ms: frame.observed_at_ms, + frame_source, + live_frame_gap, + recorded_outcome, + replayed_outcome, + export_height: replayed_export_height, + preview_height: replayed_preview_height, + session_preview_height: replayed_session_preview_height, + recorded_export_height, + recorded_preview_height, + viewport_top_y: active_session.current_viewport_top_y(), + last_commit_decision_source: telemetry.last_commit_decision_source, + last_commit_detected_motion_rows: telemetry.last_commit_detected_motion_rows, + last_block_reason: telemetry.last_block_reason, + replayed_downward_sample_registration_result: telemetry + .last_downward_sample_registration_result, + replayed_downward_sample_registration_source: telemetry + .last_downward_sample_registration_source, + replayed_downward_sample_registration_motion_rows: telemetry + .last_downward_sample_registration_motion_rows, + replayed_downward_sample_registration_provisional_viewport_top_y: telemetry + .last_downward_sample_registration_provisional_viewport_top_y, + replayed_observed_sample_registration_result: telemetry.observed_sample_registration_result, + replayed_observed_sample_registration_reason: telemetry.observed_sample_registration_reason, + replayed_observed_sample_registration_motion_rows: telemetry + .observed_sample_registration_motion_rows, + replayed_observed_sample_registration_mean_abs_diff_x100: telemetry + .observed_sample_registration_mean_abs_diff_x100, + replayed_preview_only_local_registration_result: telemetry + .preview_only_local_registration_result, + replayed_preview_only_local_registration_reason: telemetry + .preview_only_local_registration_reason, + replayed_preview_only_local_registration_motion_rows: telemetry + .preview_only_local_registration_motion_rows, + replayed_preview_only_local_registration_mean_abs_diff_x100: telemetry + .preview_only_local_registration_mean_abs_diff_x100, + replayed_downward_viewport_candidate_count: telemetry + .last_downward_viewport_candidate_count, + replayed_downward_viewport_candidates_before_prune: telemetry + .last_downward_viewport_candidates_before_prune, + replayed_downward_viewport_candidates_after_prune: telemetry + .last_downward_viewport_candidates_after_prune, + replayed_sample_eval_last_motion_rows_hint: telemetry.sample_eval_last_motion_rows_hint, + replayed_sample_eval_transient_motion_rows_hint: telemetry + .sample_eval_transient_motion_rows_hint, + replayed_sample_eval_effective_motion_rows_hint: telemetry + .sample_eval_effective_motion_rows_hint, + replayed_sample_eval_transient_burst_search_enabled: telemetry + .sample_eval_transient_burst_search_enabled, + replayed_preview_only_local_viewport_top_y: telemetry.preview_only_local_viewport_top_y, + replayed_downward_motion_rows_pending: session.scroll_capture.downward_motion_rows_pending, + replayed_input_gesture_active: session.scroll_capture.input_gesture_active, + replayed_session_preview_display_mode: active_session.preview_display_mode(), + replayed_session_preview_hinted_motion_rows_hint: None, + replayed_session_preview_hinted_frame_source: None, + replayed_overlay_preview_motion_rows_hint: session + .scroll_capture + .last_overlay_preview_motion_rows_hint, + replayed_overlay_preview_provisional_motion_rows_hint: session + .scroll_capture + .last_overlay_preview_provisional_motion_rows_hint, + replayed_overlay_preview_existing_candidate_height: session + .scroll_capture + .last_overlay_preview_existing_candidate_height, + replayed_overlay_preview_existing_candidate_motion_rows_hint: session + .scroll_capture + .last_overlay_preview_existing_candidate_motion_rows_hint, + replayed_overlay_preview_ledger_candidate_height: session + .scroll_capture + .last_overlay_preview_ledger_candidate_height, + replayed_overlay_preview_ledger_candidate_motion_rows_hint: session + .scroll_capture + .last_overlay_preview_ledger_candidate_motion_rows_hint, + replayed_overlay_preview_retained_candidate_height: session + .scroll_capture + .last_overlay_preview_retained_candidate_height, + replayed_overlay_preview_retained_candidate_motion_rows_hint: session + .scroll_capture + .last_overlay_preview_retained_candidate_motion_rows_hint, + replayed_overlay_preview_retained_hint_matches_motion_rows: session + .scroll_capture + .last_overlay_preview_retained_hint_matches_motion_rows, + replayed_overlay_preview_fresh_latest_frame_can_drive: session + .scroll_capture + .last_overlay_preview_fresh_latest_frame_can_drive, + replayed_retained_overlay_preview_height: session + .scroll_capture + .retained_overlay_preview_image + .as_ref() + .map(image::RgbaImage::height), + replayed_retained_overlay_preview_motion_rows_hint: session + .scroll_capture + .retained_overlay_preview_motion_rows_hint, + replayed_overlay_preview_strong_unresolved_registration: session + .scroll_capture + .last_overlay_preview_strong_unresolved_registration, + replayed_overlay_preview_latest_frame_present: session + .scroll_capture + .last_overlay_preview_latest_frame_present, + replayed_overlay_preview_used_provisional: session + .scroll_capture + .last_overlay_preview_used_provisional, + recorded_estimated_downward_shift_rows, + semantic_issue, + }); + replay_stats.previous_recorded_frame = Some(image); + + Ok(()) +} + +fn update_recorded_height_jumps( + replay_stats: &mut ReplayStats, + recorded_export_height: Option, + recorded_preview_height: Option, +) { + if let Some(recorded_export_height) = recorded_export_height { + if let Some(previous) = replay_stats.previous_recorded_export_height { + replay_stats.max_recorded_export_jump = replay_stats + .max_recorded_export_jump + .max(recorded_export_height.saturating_sub(previous)); + } + replay_stats.previous_recorded_export_height = Some(recorded_export_height); + } + + if let Some(recorded_preview_height) = recorded_preview_height { + if let Some(previous) = replay_stats.previous_recorded_preview_height { + replay_stats.max_recorded_preview_jump = replay_stats + .max_recorded_preview_jump + .max(recorded_preview_height.saturating_sub(previous)); + } + replay_stats.previous_recorded_preview_height = Some(recorded_preview_height); + } +} + +fn update_replayed_height_jumps( + replay_stats: &mut ReplayStats, + replayed_export_height: u32, + replayed_preview_height: u32, +) { + if let Some(previous) = replay_stats.previous_replayed_export_height { + replay_stats.max_replayed_export_jump = replay_stats + .max_replayed_export_jump + .max(replayed_export_height.saturating_sub(previous)); + } + replay_stats.previous_replayed_export_height = Some(replayed_export_height); + + if let Some(previous) = replay_stats.previous_replayed_preview_height { + replay_stats.max_replayed_preview_jump = replay_stats + .max_replayed_preview_jump + .max(replayed_preview_height.saturating_sub(previous)); + } + replay_stats.previous_replayed_preview_height = Some(replayed_preview_height); +} + +fn update_live_frame_gap( + replay_stats: &mut ReplayStats, + frame_source: RecordedScrollCaptureReplayFrameSource, +) -> Option { + match frame_source { + RecordedScrollCaptureReplayFrameSource::LiveStream { frame_seq } => { + let gap = replay_stats + .previous_live_frame_seq + .map(|previous| frame_seq.saturating_sub(previous)) + .unwrap_or(1); + replay_stats.previous_live_frame_seq = Some(frame_seq); + Some(gap) + }, + RecordedScrollCaptureReplayFrameSource::Worker { .. } => None, + } +} + +fn finalize_replay_summary( + trace: LoadedScrollCaptureLiveTrace, + session: &OverlaySession, + replay_stats: ReplayStats, + replay_mode: RecordedScrollCaptureReplayMode, +) -> Result { + let final_session = session.scroll_capture.session.as_ref().ok_or_else(|| { + eyre!("scroll-capture session missing after replaying recorded live trace") + })?; + let first_outcome_divergence_frame = replay_stats + .step_results + .iter() + .find(|step| !recorded_step_outcome_matches_replayed(step)) + .map(|step| step.frame_index); + let first_export_height_drift_frame = replay_stats + .step_results + .iter() + .find(|step| { + step.recorded_export_height.is_some_and(|recorded| recorded != step.export_height) + }) + .map(|step| step.frame_index); + let first_preview_height_drift_frame = replay_stats + .step_results + .iter() + .find(|step| { + step.recorded_preview_height.is_some_and(|recorded| recorded != step.preview_height) + }) + .map(|step| step.frame_index); + let first_semantic_issue_frame = replay_stats + .step_results + .iter() + .find(|step| step.semantic_issue.is_some()) + .map(|step| step.frame_index); + let first_missed_downward_motion_frame = replay_stats + .step_results + .iter() + .find(|step| { + matches!( + step.semantic_issue, + Some(RecordedScrollCaptureSemanticIssue::MissedDownwardMotion) + ) + }) + .map(|step| step.frame_index); + let first_underconsumed_downward_motion_frame = replay_stats + .step_results + .iter() + .find(|step| { + matches!( + step.semantic_issue, + Some(RecordedScrollCaptureSemanticIssue::UnderconsumedDownwardMotion) + ) + }) + .map(|step| step.frame_index); + let first_growth_overshoot_frame = replay_stats + .step_results + .iter() + .find(|step| { + matches!( + step.semantic_issue, + Some(RecordedScrollCaptureSemanticIssue::GrowthExceedsRecordedShift) + ) + }) + .map(|step| step.frame_index); + + Ok(RecordedScrollCaptureReplaySummary { + replay_mode, + trace_id: trace.manifest.trace_id.clone(), + manifest_path: trace.manifest_path.clone(), + final_export_height: final_session.export_image().height(), + final_preview_height: session + .scroll_capture_preview_dimensions() + .map_or(final_session.preview_image().height(), |dimensions| dimensions[1]), + final_viewport_top_y: final_session.current_viewport_top_y(), + recorded_final_export_height: trace + .manifest + .final_snapshot + .as_ref() + .and_then(|snapshot| snapshot.export_dimensions) + .map(|dimensions| dimensions[1]), + recorded_final_preview_height: trace + .manifest + .final_snapshot + .as_ref() + .and_then(|snapshot| snapshot.preview_dimensions) + .map(|dimensions| dimensions[1]), + final_preview_path: trace + .manifest + .final_preview_path + .as_deref() + .map(|path| trace.resolve_frame_path(path)), + final_export_path: trace + .manifest + .final_export_path + .as_deref() + .map(|path| trace.resolve_frame_path(path)), + first_outcome_divergence_frame, + first_export_height_drift_frame, + first_preview_height_drift_frame, + max_recorded_committed_growth_rows: replay_stats.max_recorded_committed_growth_rows, + max_replayed_committed_growth_rows: replay_stats.max_replayed_committed_growth_rows, + max_recorded_export_jump: replay_stats.max_recorded_export_jump, + max_recorded_preview_jump: replay_stats.max_recorded_preview_jump, + max_replayed_export_jump: replay_stats.max_replayed_export_jump, + max_replayed_preview_jump: replay_stats.max_replayed_preview_jump, + first_semantic_issue_frame, + first_missed_downward_motion_frame, + first_underconsumed_downward_motion_frame, + first_growth_overshoot_frame, + step_results: replay_stats.step_results, + }) +} + +fn estimate_recorded_downward_shift_rows(previous: &RgbaImage, current: &RgbaImage) -> Option { + if previous.dimensions() != current.dimensions() { + return None; + } + let (width, height) = previous.dimensions(); + if width < 2 || height < 3 { + return None; + } + + let margin_x = (width / 8).min(width.saturating_sub(2) / 2); + let start_x = margin_x; + let end_x = width.saturating_sub(margin_x).max(start_x + 1); + let x_step = ((end_x.saturating_sub(start_x)) / 48).max(1); + let y_step = 2_u32; + let max_shift = height.saturating_sub(1).min(96); + + let mut best_shift = 0_u32; + let mut best_score = overlap_abs_diff(previous, current, 0, start_x, end_x, x_step, y_step)?; + + for shift in 1..=max_shift { + let Some(score) = + overlap_abs_diff(previous, current, shift, start_x, end_x, x_step, y_step) + else { + continue; + }; + if score < best_score { + best_score = score; + best_shift = shift; + } + } + + Some(best_shift) +} + +fn overlap_abs_diff( + previous: &RgbaImage, + current: &RgbaImage, + shift: u32, + start_x: u32, + end_x: u32, + x_step: u32, + y_step: u32, +) -> Option { + let height = previous.height(); + if shift >= height { + return None; + } + let overlap_height = height - shift; + if overlap_height < 2 { + return None; + } + + let mut sum = 0_u64; + let mut samples = 0_u64; + let mut y = 0_u32; + while y < overlap_height { + let mut x = start_x; + while x < end_x { + let prev = previous.get_pixel(x, y + shift); + let curr = current.get_pixel(x, y); + let prev_luma = u16::from(prev[0]) + u16::from(prev[1]) + u16::from(prev[2]); + let curr_luma = u16::from(curr[0]) + u16::from(curr[1]) + u16::from(curr[2]); + sum += u64::from(prev_luma.abs_diff(curr_luma)); + samples += 1; + x = x.saturating_add(x_step); + } + y = y.saturating_add(y_step); + } + + if samples == 0 { + return None; + } + + Some(sum / samples) +} + +fn classify_recorded_semantic_issue( + recorded_outcome: &RecordedScrollCaptureReplayRecordedOutcome, + recorded_estimated_downward_shift_rows: Option, +) -> Option { + let shift = recorded_estimated_downward_shift_rows?; + if shift < 4 { + return None; + } + + match recorded_outcome { + RecordedScrollCaptureReplayRecordedOutcome::NoChange + | RecordedScrollCaptureReplayRecordedOutcome::PreviewUpdated => { + Some(RecordedScrollCaptureSemanticIssue::MissedDownwardMotion) + }, + RecordedScrollCaptureReplayRecordedOutcome::Committed { + direction: "down", + growth_rows, + } if growth_rows.saturating_mul(2).saturating_add(2) < shift => { + Some(RecordedScrollCaptureSemanticIssue::UnderconsumedDownwardMotion) + }, + RecordedScrollCaptureReplayRecordedOutcome::Committed { + direction: "down", + growth_rows, + } if *growth_rows > shift.saturating_add(8) => { + Some(RecordedScrollCaptureSemanticIssue::GrowthExceedsRecordedShift) + }, + _ => None, + } +} + +#[cfg(target_os = "macos")] +fn replay_frame_source(frame_source: ScrollCaptureTraceFrameSource) -> ScrollCaptureFrameSource { + match frame_source { + ScrollCaptureTraceFrameSource::Worker { .. } => { + unreachable!("macOS live traces should not contain worker-backed scroll frames") + }, + ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { + ScrollCaptureFrameSource::LiveStream { frame_seq } + }, + } +} + +#[cfg(not(target_os = "macos"))] +fn replay_frame_source(frame_source: ScrollCaptureTraceFrameSource) -> ScrollCaptureFrameSource { + match frame_source { + ScrollCaptureTraceFrameSource::Worker { request_id } => { + ScrollCaptureFrameSource::Worker { request_id } + }, + ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { + let _ = frame_seq; + unreachable!("non-macOS replay should not receive live-stream scroll frames") + }, + } +} + +fn recorded_outcome_matches_replayed( + recorded: &RecordedScrollCaptureReplayRecordedOutcome, + replayed: ScrollCaptureReplayOutcome, +) -> bool { + match (recorded, replayed) { + ( + RecordedScrollCaptureReplayRecordedOutcome::NoChange, + ScrollCaptureReplayOutcome::NoChange, + ) => true, + ( + RecordedScrollCaptureReplayRecordedOutcome::PreviewUpdated, + ScrollCaptureReplayOutcome::PreviewUpdated, + ) => true, + ( + RecordedScrollCaptureReplayRecordedOutcome::Unsupported { direction }, + ScrollCaptureReplayOutcome::UnsupportedUp, + ) => *direction == "up", + ( + RecordedScrollCaptureReplayRecordedOutcome::Committed { direction, growth_rows }, + ScrollCaptureReplayOutcome::CommittedDown { growth_rows: replayed_growth_rows }, + ) => *direction == "down" && *growth_rows == replayed_growth_rows, + (RecordedScrollCaptureReplayRecordedOutcome::Error { .. }, _) => false, + _ => false, + } +} + +fn recorded_step_outcome_matches_replayed(step: &RecordedScrollCaptureReplayStepResult) -> bool { + if recorded_outcome_matches_replayed(&step.recorded_outcome, step.replayed_outcome) { + return true; + } + + matches!( + (&step.recorded_outcome, step.replayed_outcome), + ( + RecordedScrollCaptureReplayRecordedOutcome::NoChange, + ScrollCaptureReplayOutcome::PreviewUpdated, + ) | ( + RecordedScrollCaptureReplayRecordedOutcome::PreviewUpdated, + ScrollCaptureReplayOutcome::NoChange, + ) + ) && step.recorded_export_height == Some(step.export_height) + && step.recorded_preview_height == Some(step.preview_height) +} + +fn replay_monitor_from_trace(trace: &LoadedScrollCaptureLiveTrace) -> MonitorRect { + MonitorRect { + id: trace.manifest.monitor.id, + origin: GlobalPoint::new(trace.manifest.monitor.origin_x, trace.manifest.monitor.origin_y), + width: trace.manifest.monitor.width, + height: trace.manifest.monitor.height, + scale_factor_x1000: trace.manifest.monitor.scale_factor_x1000, + } +} + +fn replay_capture_rect_from_trace(trace: &LoadedScrollCaptureLiveTrace) -> RectPoints { + RectPoints::new( + trace.manifest.capture_rect_pixels.x, + trace.manifest.capture_rect_pixels.y, + trace.manifest.capture_rect_pixels.width, + trace.manifest.capture_rect_pixels.height, + ) +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + path::PathBuf, + process, + time::{Duration, Instant}, + }; + + use image::{Rgba, RgbaImage}; + + use super::{ + RecordedScrollCaptureReplayMode, replay_recorded_scroll_capture_trace, + replay_recorded_scroll_capture_trace_with_mode, + }; + use crate::overlay::{ + GlobalPoint, MonitorRect, OverlaySession, RectPoints, ScrollCaptureFrameSource, + trace_recording::{ + ScrollCaptureTraceFrameRecord, ScrollCaptureTraceInputRecord, + ScrollCaptureTraceRecorder, ScrollCaptureTraceSessionSnapshot, + }, + }; + use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; + + fn temp_trace_root() -> PathBuf { + let root = std::env::temp_dir().join(format!( + "rsnap-recorded-trace-replay-test-{}-{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis(), + process::id() + )); + let _ = fs::remove_dir_all(&root); + + root + } + + fn monitor() -> MonitorRect { + MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + } + } + + fn capture_rect() -> RectPoints { + RectPoints::new(100, 120, 3, 5) + } + + fn large_capture_rect() -> RectPoints { + RectPoints::new(100, 120, 256, 120) + } + + fn make_window(rows: &[[u8; 4]], start: usize) -> RgbaImage { + let mut image = RgbaImage::new(3, 5); + + for (y, row) in rows[start..start + 5].iter().enumerate() { + for x in 0..3 { + image.put_pixel(x, y as u32, Rgba(*row)); + } + } + + image + } + + fn make_sparse_textlike_window(width: u32, height: u32, start_row: u32) -> RgbaImage { + let stripe_x = 104_u32.min(width.saturating_sub(1)); + let mut image = RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255])); + + for y in 0..height { + let document_row = start_row.saturating_add(y); + let shade = ((document_row.saturating_mul(17)) % 180) as u8; + + for x in stripe_x..stripe_x.saturating_add(6).min(width) { + image.put_pixel(x, y, Rgba([shade, shade, shade, 255])); + } + for x in stripe_x.saturating_add(10)..stripe_x.saturating_add(13).min(width) { + if document_row % 19 < 9 { + image.put_pixel(x, y, Rgba([40, 40, 40, 255])); + } + } + } + + image + } + + #[test] + fn replay_recorded_live_trace_round_trips_one_commit() { + let rows = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + ]; + let base_frame = make_window(&rows, 0); + let next_frame = make_window(&rows, 1); + let mut session = OverlaySession::new(); + let root = temp_trace_root(); + let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( + root, + monitor(), + capture_rect(), + 320, + &base_frame, + ) + .unwrap(); + let manifest_path = recorder.manifest_path().to_path_buf(); + let started_at = Instant::now(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor()); + session.scroll_capture.capture_rect_pixels = Some(capture_rect()); + session.scroll_capture.session = Some(ScrollSession::new(base_frame.clone(), 320).unwrap()); + + session.apply_external_scroll_input_delta_y( + 150.0, + 160.0, + 4.0, + true, + false, + started_at + Duration::from_millis(10), + ); + recorder.record_replayed_input(ScrollCaptureTraceInputRecord { + seq: 1, + cursor_global: (150.0, 160.0), + delta_y: 4.0, + gesture_active: true, + gesture_ended: false, + recorded_age: Duration::from_millis(2), + applied_at: started_at + Duration::from_millis(10), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + Some(ScrollDirection::Down), + true, + 4.0, + Some(2), + ), + }); + let outcome = session + .observe_scroll_capture_frame_at( + next_frame.clone(), + started_at + Duration::from_millis(20), + ) + .transpose() + .unwrap() + .unwrap(); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: &next_frame, + source: ScrollCaptureFrameSource::LiveStream { frame_seq: 9 }, + allow_stale_input: false, + prior_block_reason: None, + observed_at: started_at + Duration::from_millis(20), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + session.scroll_capture.input_direction, + session.scroll_capture.input_gesture_active, + session.scroll_capture.downward_motion_rows_pending, + Some(0), + ), + outcome: &Ok(outcome), + }); + + drop(recorder); + + let summary = replay_recorded_scroll_capture_trace(&manifest_path).unwrap(); + + assert_eq!(summary.step_results.len(), 1); + assert_eq!( + summary.step_results[0].recorded_outcome, + super::RecordedScrollCaptureReplayRecordedOutcome::Committed { + direction: "down", + growth_rows: 1, + } + ); + assert_eq!( + summary.step_results[0].replayed_outcome, + super::ScrollCaptureReplayOutcome::CommittedDown { growth_rows: 1 } + ); + assert_eq!(summary.final_export_height, 6); + assert_eq!(summary.max_replayed_export_jump, 0); + assert_eq!(summary.max_replayed_preview_jump, 0); + } + + #[test] + fn replay_recorded_live_trace_round_trips_one_commit_forces_worker_pairwise() { + let base_frame = make_sparse_textlike_window(256, 120, 0); + let next_frame = make_sparse_textlike_window(256, 120, 9); + let mut session = OverlaySession::new(); + let root = temp_trace_root(); + let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( + root, + monitor(), + large_capture_rect(), + 320, + &base_frame, + ) + .unwrap(); + let manifest_path = recorder.manifest_path().to_path_buf(); + let started_at = Instant::now(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor()); + session.scroll_capture.capture_rect_pixels = Some(large_capture_rect()); + session.scroll_capture.session = Some(ScrollSession::new(base_frame.clone(), 320).unwrap()); + + session.apply_external_scroll_input_delta_y( + 150.0, + 160.0, + 9.0, + true, + false, + started_at + Duration::from_millis(10), + ); + recorder.record_replayed_input(ScrollCaptureTraceInputRecord { + seq: 1, + cursor_global: (150.0, 160.0), + delta_y: 4.0, + gesture_active: true, + gesture_ended: false, + recorded_age: Duration::from_millis(2), + applied_at: started_at + Duration::from_millis(10), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + Some(ScrollDirection::Down), + true, + 9.0, + Some(2), + ), + }); + let outcome = session + .observe_scroll_capture_frame_at( + next_frame.clone(), + started_at + Duration::from_millis(20), + ) + .transpose() + .unwrap() + .unwrap(); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: &next_frame, + source: ScrollCaptureFrameSource::LiveStream { frame_seq: 9 }, + allow_stale_input: false, + prior_block_reason: None, + observed_at: started_at + Duration::from_millis(20), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + session.scroll_capture.input_direction, + session.scroll_capture.input_gesture_active, + session.scroll_capture.downward_motion_rows_pending, + Some(0), + ), + outcome: &Ok(outcome), + }); + + drop(recorder); + + let ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows } = + outcome + else { + panic!("expected recorded-source setup to commit one downward growth step"); + }; + + let summary = replay_recorded_scroll_capture_trace_with_mode( + &manifest_path, + RecordedScrollCaptureReplayMode::ForceWorkerPairwise, + ) + .unwrap(); + + assert_eq!(summary.replay_mode, RecordedScrollCaptureReplayMode::ForceWorkerPairwise); + assert_eq!(summary.step_results.len(), 1); + assert_eq!( + summary.step_results[0].recorded_outcome, + super::RecordedScrollCaptureReplayRecordedOutcome::Committed { + direction: "down", + growth_rows, + } + ); + assert_eq!( + summary.step_results[0].replayed_outcome, + super::ScrollCaptureReplayOutcome::CommittedDown { growth_rows } + ); + assert_eq!(summary.final_export_height, base_frame.height() + growth_rows); + assert_eq!(summary.max_replayed_export_jump, 0); + assert_eq!(summary.max_replayed_preview_jump, 0); + } + + #[test] + fn estimated_downward_shift_rows_detects_simple_shift() { + let rows = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + [90, 0, 0, 255], + [100, 0, 0, 255], + ]; + let previous = make_window(&rows, 0); + let current = make_window(&rows, 2); + + assert_eq!(super::estimate_recorded_downward_shift_rows(&previous, ¤t), Some(2)); + } + + #[test] + fn classify_replayed_outcome_upgrades_no_change_when_only_preview_grew() { + assert_eq!( + super::classify_replayed_outcome( + ScrollObserveOutcome::NoChange, + Some(100), + 100, + Some(120), + 145, + ), + super::ScrollCaptureReplayOutcome::PreviewUpdated + ); + } + + #[test] + fn classify_replayed_outcome_keeps_no_change_when_export_changed() { + assert_eq!( + super::classify_replayed_outcome( + ScrollObserveOutcome::NoChange, + Some(100), + 101, + Some(120), + 145, + ), + super::ScrollCaptureReplayOutcome::NoChange + ); + } + + #[test] + fn recorded_step_outcome_match_ignores_no_change_vs_preview_updated_when_heights_align() { + let step = super::RecordedScrollCaptureReplayStepResult { + frame_index: 0, + frame_path: String::new(), + observed_at_ms: 0, + frame_source: super::RecordedScrollCaptureReplayFrameSource::LiveStream { + frame_seq: 1, + }, + live_frame_gap: Some(1), + recorded_outcome: super::RecordedScrollCaptureReplayRecordedOutcome::NoChange, + replayed_outcome: super::ScrollCaptureReplayOutcome::PreviewUpdated, + export_height: 100, + preview_height: 148, + session_preview_height: 148, + recorded_export_height: Some(100), + recorded_preview_height: Some(148), + viewport_top_y: 0, + last_commit_decision_source: None, + last_commit_detected_motion_rows: None, + last_block_reason: None, + replayed_downward_sample_registration_result: None, + replayed_downward_sample_registration_source: None, + replayed_downward_sample_registration_motion_rows: None, + replayed_downward_sample_registration_provisional_viewport_top_y: None, + replayed_observed_sample_registration_result: None, + replayed_observed_sample_registration_reason: None, + replayed_observed_sample_registration_motion_rows: None, + replayed_observed_sample_registration_mean_abs_diff_x100: None, + replayed_preview_only_local_registration_result: None, + replayed_preview_only_local_registration_reason: None, + replayed_preview_only_local_registration_motion_rows: None, + replayed_preview_only_local_registration_mean_abs_diff_x100: None, + replayed_downward_viewport_candidate_count: None, + replayed_downward_viewport_candidates_before_prune: None, + replayed_downward_viewport_candidates_after_prune: None, + replayed_sample_eval_last_motion_rows_hint: None, + replayed_sample_eval_transient_motion_rows_hint: None, + replayed_sample_eval_effective_motion_rows_hint: None, + replayed_sample_eval_transient_burst_search_enabled: false, + replayed_preview_only_local_viewport_top_y: None, + replayed_downward_motion_rows_pending: 0.0, + replayed_input_gesture_active: false, + replayed_session_preview_display_mode: "committed", + replayed_session_preview_hinted_motion_rows_hint: None, + replayed_session_preview_hinted_frame_source: None, + replayed_overlay_preview_motion_rows_hint: None, + replayed_overlay_preview_provisional_motion_rows_hint: None, + replayed_overlay_preview_existing_candidate_height: None, + replayed_overlay_preview_existing_candidate_motion_rows_hint: None, + replayed_overlay_preview_ledger_candidate_height: None, + replayed_overlay_preview_ledger_candidate_motion_rows_hint: None, + replayed_overlay_preview_retained_candidate_height: None, + replayed_overlay_preview_retained_candidate_motion_rows_hint: None, + replayed_overlay_preview_retained_hint_matches_motion_rows: false, + replayed_overlay_preview_fresh_latest_frame_can_drive: false, + replayed_retained_overlay_preview_height: None, + replayed_retained_overlay_preview_motion_rows_hint: None, + replayed_overlay_preview_strong_unresolved_registration: false, + replayed_overlay_preview_latest_frame_present: false, + replayed_overlay_preview_used_provisional: false, + recorded_estimated_downward_shift_rows: None, + semantic_issue: None, + }; + + assert!(super::recorded_step_outcome_matches_replayed(&step)); + } + + #[test] + fn recorded_step_outcome_match_keeps_divergence_when_only_outcome_label_matches_bad_heights() { + let step = super::RecordedScrollCaptureReplayStepResult { + frame_index: 0, + frame_path: String::new(), + observed_at_ms: 0, + frame_source: super::RecordedScrollCaptureReplayFrameSource::LiveStream { + frame_seq: 1, + }, + live_frame_gap: Some(1), + recorded_outcome: super::RecordedScrollCaptureReplayRecordedOutcome::NoChange, + replayed_outcome: super::ScrollCaptureReplayOutcome::PreviewUpdated, + export_height: 100, + preview_height: 148, + session_preview_height: 148, + recorded_export_height: Some(100), + recorded_preview_height: Some(147), + viewport_top_y: 0, + last_commit_decision_source: None, + last_commit_detected_motion_rows: None, + last_block_reason: None, + replayed_downward_sample_registration_result: None, + replayed_downward_sample_registration_source: None, + replayed_downward_sample_registration_motion_rows: None, + replayed_downward_sample_registration_provisional_viewport_top_y: None, + replayed_observed_sample_registration_result: None, + replayed_observed_sample_registration_reason: None, + replayed_observed_sample_registration_motion_rows: None, + replayed_observed_sample_registration_mean_abs_diff_x100: None, + replayed_preview_only_local_registration_result: None, + replayed_preview_only_local_registration_reason: None, + replayed_preview_only_local_registration_motion_rows: None, + replayed_preview_only_local_registration_mean_abs_diff_x100: None, + replayed_downward_viewport_candidate_count: None, + replayed_downward_viewport_candidates_before_prune: None, + replayed_downward_viewport_candidates_after_prune: None, + replayed_sample_eval_last_motion_rows_hint: None, + replayed_sample_eval_transient_motion_rows_hint: None, + replayed_sample_eval_effective_motion_rows_hint: None, + replayed_sample_eval_transient_burst_search_enabled: false, + replayed_preview_only_local_viewport_top_y: None, + replayed_downward_motion_rows_pending: 0.0, + replayed_input_gesture_active: false, + replayed_session_preview_display_mode: "committed", + replayed_session_preview_hinted_motion_rows_hint: None, + replayed_session_preview_hinted_frame_source: None, + replayed_overlay_preview_motion_rows_hint: None, + replayed_overlay_preview_provisional_motion_rows_hint: None, + replayed_overlay_preview_existing_candidate_height: None, + replayed_overlay_preview_existing_candidate_motion_rows_hint: None, + replayed_overlay_preview_ledger_candidate_height: None, + replayed_overlay_preview_ledger_candidate_motion_rows_hint: None, + replayed_overlay_preview_retained_candidate_height: None, + replayed_overlay_preview_retained_candidate_motion_rows_hint: None, + replayed_overlay_preview_retained_hint_matches_motion_rows: false, + replayed_overlay_preview_fresh_latest_frame_can_drive: false, + replayed_retained_overlay_preview_height: None, + replayed_retained_overlay_preview_motion_rows_hint: None, + replayed_overlay_preview_strong_unresolved_registration: false, + replayed_overlay_preview_latest_frame_present: false, + replayed_overlay_preview_used_provisional: false, + recorded_estimated_downward_shift_rows: None, + semantic_issue: None, + }; + + assert!(!super::recorded_step_outcome_matches_replayed(&step)); + } + + #[test] + fn semantic_issue_flags_missed_downward_motion_when_shift_exists_without_growth() { + assert_eq!( + super::classify_recorded_semantic_issue( + &super::RecordedScrollCaptureReplayRecordedOutcome::PreviewUpdated, + Some(12), + ), + Some(super::RecordedScrollCaptureSemanticIssue::MissedDownwardMotion) + ); + } +} diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index f2c9a620..6dcb78cb 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -3,21 +3,43 @@ use std::time::Instant; use color_eyre::Result; use image::RgbaImage; -#[cfg(not(target_os = "macos"))] use crate::overlay::SCROLL_CAPTURE_SAMPLE_INTERVAL; #[cfg(target_os = "macos")] -use crate::overlay::{LiveStreamStaleGrace, SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES}; -#[cfg(any(not(target_os = "macos"), test))] +use crate::overlay::session_state::ScrollCaptureLiveFrame; +#[cfg(target_os = "macos")] +use crate::overlay::{ + LiveStreamStaleGrace, SCROLL_CAPTURE_DUPLICATE_STREAM_REFRESH_INTERVAL, + SCROLL_CAPTURE_DUPLICATE_STREAM_STALL_THRESHOLD, SCROLL_CAPTURE_INPUT_FRESHNESS, + SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, + SCROLL_CAPTURE_STREAM_EVENT_FALLBACK_POLL_INTERVAL, SCROLL_CAPTURE_STREAM_POLL_INTERVAL, +}; use crate::overlay::{MonitorRect, RectPoints}; use crate::overlay::{ - OverlayControl, OverlaySession, ScrollCaptureFrameSource, ScrollObserveOutcome, ScrollSession, + OverlayControl, OverlaySession, ScrollCaptureFrameSource, ScrollCaptureTraceFrameRecord, + ScrollCaptureTraceInputRecord, ScrollObserveOutcome, ScrollSession, }; #[cfg(target_os = "macos")] use crate::scroll_capture::ScrollDirection; -#[cfg(not(target_os = "macos"))] +#[cfg(target_os = "macos")] +use crate::scroll_capture::{scroll_capture_fingerprint, scroll_capture_fingerprint_delta}; use crate::worker::WorkerRequestSendError; impl OverlaySession { + #[cfg(target_os = "macos")] + fn should_use_scroll_capture_worker_sampling(&self) -> bool { + if !cfg!(test) { + return true; + } + + #[cfg(test)] + { + return self.scroll_capture.force_worker_sampling_in_tests; + } + + #[allow(unreachable_code)] + false + } + pub(super) fn maybe_tick_scroll_capture(&mut self) { if !self.scroll_capture.active || self.scroll_capture.paused { return; @@ -25,61 +47,114 @@ impl OverlaySession { #[cfg(target_os = "macos")] { - self.sync_scroll_overlay_mouse_passthrough_window(Instant::now()); + let now = Instant::now(); - let _ = self.try_consume_scroll_stream_frame(); + self.sync_scroll_overlay_mouse_passthrough_window(now); + self.drain_external_scroll_input_events_through(now); + if self.should_use_scroll_capture_worker_sampling() { + self.request_scroll_capture_worker_sample_at(now); + + return; + } + self.poll_scroll_stream_fallback_if_due(now); + if self.scroll_capture.live_stream.is_some() + && self.scroll_capture.last_stream_poll_at.map_or(true, |last| { + now.saturating_duration_since(last) >= SCROLL_CAPTURE_STREAM_POLL_INTERVAL + }) { + self.scroll_capture.last_stream_poll_at = Some(now); + let _ = self.try_consume_scroll_stream_frame(); + } } #[cfg(not(target_os = "macos"))] { - if self.scroll_capture.inflight_request_id.is_some() { - return; - } + self.request_scroll_capture_worker_sample_at(Instant::now()); + } + } - let now = Instant::now(); - let Some(next_sample_at) = self.scroll_capture.next_sample_at else { - self.scroll_capture.next_sample_at = Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL); + fn request_scroll_capture_worker_sample_at(&mut self, now: Instant) { + if self.scroll_capture.inflight_request_id.is_some() { + return; + } - return; - }; + let Some(next_sample_at) = self.scroll_capture.next_sample_at else { + self.scroll_capture.next_sample_at = Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL); - if now < next_sample_at { - return; - } + return; + }; - let Some(monitor) = self.scroll_capture.monitor else { - self.scroll_capture_set_error("Scroll capture lost its monitor."); + if now < next_sample_at { + return; + } - return; - }; - let Some(capture_rect) = self.scroll_capture.capture_rect_pixels else { - self.scroll_capture_set_error("Scroll capture lost its region."); + let Some(monitor) = self.scroll_capture.monitor else { + self.scroll_capture_set_error("Scroll capture lost its monitor."); - return; - }; - let Some(worker) = self.worker.as_ref() else { - self.scroll_capture_set_error("Scroll capture worker is unavailable."); + return; + }; + let Some(capture_rect) = self.scroll_capture.capture_rect_pixels else { + self.scroll_capture_set_error("Scroll capture lost its region."); - return; - }; - let request_id = self.scroll_capture.next_request_id.wrapping_add(1); - - match worker.request_capture_monitor_region(monitor, capture_rect, request_id) { - Ok(()) => { - self.scroll_capture.next_request_id = request_id; - self.scroll_capture.inflight_request_id = Some(request_id); - self.scroll_capture.next_sample_at = Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL); - }, - Err(WorkerRequestSendError::Full) => { - self.scroll_capture.next_sample_at = - Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL.saturating_mul(2)); - }, - Err(WorkerRequestSendError::Disconnected) => { - self.scroll_capture_set_error("Scroll capture worker disconnected."); - }, - } + return; + }; + let Some(worker) = self.worker.as_ref() else { + self.scroll_capture_set_error("Scroll capture worker is unavailable."); + + return; + }; + let request_id = self.scroll_capture.next_request_id.wrapping_add(1); + + match worker.request_capture_monitor_region(monitor, capture_rect, request_id) { + Ok(()) => { + self.scroll_capture.next_request_id = request_id; + self.scroll_capture.inflight_request_id = Some(request_id); + #[cfg(target_os = "macos")] + { + self.scroll_capture.inflight_request_observation = + Some(crate::overlay::session_state::InflightScrollCaptureObservation { + was_observable: self + .scroll_capture_observation_block_reason_at(now) + .is_none(), + external_input_seq: self.scroll_capture.last_external_scroll_input_seq, + input_direction: self.scroll_capture.input_direction, + }); + } + self.scroll_capture.next_sample_at = Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL); + }, + Err(WorkerRequestSendError::Full) => { + self.scroll_capture.next_sample_at = + Some(now + SCROLL_CAPTURE_SAMPLE_INTERVAL.saturating_mul(2)); + }, + Err(WorkerRequestSendError::Disconnected) => { + self.scroll_capture_set_error("Scroll capture worker disconnected."); + }, } } + #[cfg(target_os = "macos")] + fn schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( + &mut self, + now: Instant, + why: &'static str, + ) { + let fresh_downward_input = self.scroll_capture.input_direction + == Some(ScrollDirection::Down) + && self.scroll_capture.input_direction_at.is_some_and(|input_direction_at| { + now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS + }); + if !fresh_downward_input { + return; + } + + self.scroll_capture.next_sample_at = Some(now); + tracing::info!( + op = "scroll_capture.worker_retry_scheduled_immediately", + reason = why, + last_external_scroll_input_seq = self.scroll_capture.last_external_scroll_input_seq, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, + "Scheduled an immediate worker retry because fresh downward input was still active." + ); + } + #[cfg(target_os = "macos")] pub(super) fn try_consume_scroll_stream_frame(&mut self) -> bool { let Some(monitor) = self.scroll_capture.monitor else { @@ -92,30 +167,96 @@ impl OverlaySession { return true; }; + let query_started_at = Instant::now(); + let force_refresh = self.scroll_capture_should_force_stream_refresh_at(query_started_at); + let allow_stale_refresh = + self.scroll_capture_should_schedule_stale_stream_refresh_at(query_started_at); + let fresh_downward_backlog = + self.scroll_capture_has_fresh_downward_backlog_at(query_started_at); let Some(live_stream) = self.scroll_capture.live_stream.as_mut() else { return false; }; let last_frame_seq = self.scroll_capture.last_stream_frame_seq; - let Some(frames) = - live_stream.ordered_rgba_regions_after_seq(monitor, capture_rect, last_frame_seq) - else { + let log_stream_frame_empty = |query_ms: u64, refresh_scheduled: bool| { + if query_ms >= 16 { + tracing::warn!( + op = "scroll_capture.stream_frame_query_slow", + last_frame_seq, + query_ms, + refresh_scheduled, + stale_refresh_suppressed = !allow_stale_refresh, + force_refresh, + result = "empty", + "Slow nonblocking live-stream query delayed scroll-capture observation." + ); + } tracing::info!( op = "scroll_capture.stream_frame_empty", last_frame_seq, + query_ms, + refresh_scheduled, + stale_refresh_suppressed = !allow_stale_refresh, + force_refresh, "Did not receive a newer live-stream frame for scroll-capture observation." ); + }; + let Some(frames) = live_stream.ordered_rgba_regions_after_seq_nonblocking( + monitor, + capture_rect, + last_frame_seq, + ) else { + let query_ms = + u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let refresh_scheduled = if allow_stale_refresh { + live_stream.refresh_monitor_nonblocking_if_stale( + monitor, + last_frame_seq, + force_refresh, + ) + } else { + false + }; + if refresh_scheduled && fresh_downward_backlog { + self.scroll_capture.pending_post_stall_burst_after_seq = Some(last_frame_seq); + } + + log_stream_frame_empty(query_ms, refresh_scheduled); return false; }; let Some(newest_frame_seq) = frames.last().map(|frame| frame.frame_seq) else { - tracing::info!( - op = "scroll_capture.stream_frame_empty", - last_frame_seq, - "Did not receive a newer live-stream frame for scroll-capture observation." - ); + let query_ms = + u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let refresh_scheduled = if allow_stale_refresh { + live_stream.refresh_monitor_nonblocking_if_stale( + monitor, + last_frame_seq, + force_refresh, + ) + } else { + false + }; + if refresh_scheduled && fresh_downward_backlog { + self.scroll_capture.pending_post_stall_burst_after_seq = Some(last_frame_seq); + } + + log_stream_frame_empty(query_ms, refresh_scheduled); return false; }; + let query_ms = u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + + if query_ms >= 16 { + tracing::warn!( + op = "scroll_capture.stream_frame_query_slow", + last_frame_seq, + query_ms, + result = "ready", + frame_seq = newest_frame_seq, + frame_count = frames.len(), + "Slow nonblocking live-stream query delayed scroll-capture observation." + ); + } tracing::info!( op = "scroll_capture.stream_frame_ready", @@ -123,30 +264,56 @@ impl OverlaySession { frame_seq = newest_frame_seq, frame_gap = newest_frame_seq.saturating_sub(last_frame_seq), frame_count = frames.len(), + query_ms, "Pulled live-stream frame for scroll-capture observation." ); for frame in frames { - self.drain_external_scroll_input_events_through(frame.captured_at); - - self.scroll_capture.last_stream_frame_seq = frame.frame_seq; - - self.handle_scroll_capture_frame( - frame.image, - ScrollCaptureFrameSource::LiveStream { frame_seq: frame.frame_seq }, - false, - frame.captured_at, - ); + self.push_scroll_capture_live_frame(ScrollCaptureLiveFrame { + frame_seq: frame.frame_seq, + captured_at: frame.captured_at, + image: frame.image, + }); } + self.consume_scroll_capture_backlog(usize::MAX); + true } #[cfg(target_os = "macos")] /// Consumes any queued macOS live-stream frames for scroll capture. pub fn handle_scroll_stream_frame_ready(&mut self) -> OverlayControl { + if !cfg!(test) { + return OverlayControl::Continue; + } + if self.scroll_capture.active && !self.scroll_capture.paused { + if self.should_use_scroll_capture_worker_sampling() { + return OverlayControl::Continue; + } let _ = self.try_consume_scroll_stream_frame(); + self.consume_scroll_capture_backlog(usize::MAX); + } + + OverlayControl::Continue + } + + #[cfg(target_os = "macos")] + /// Drains queued external scroll input and opportunistically polls the stream fallback path. + pub fn handle_scroll_input_ready(&mut self) -> OverlayControl { + if self.scroll_capture.active && !self.scroll_capture.paused { + let now = Instant::now(); + + self.sync_scroll_overlay_mouse_passthrough_window(now); + self.drain_external_scroll_input_events_through(now); + if self.should_use_scroll_capture_worker_sampling() { + self.request_scroll_capture_worker_sample_at(now); + + return OverlayControl::Continue; + } + self.poll_scroll_stream_fallback_if_due(now); + self.consume_scroll_capture_backlog(usize::MAX); } OverlayControl::Continue @@ -170,7 +337,8 @@ impl OverlaySession { continue; } - let inferred_direction = Self::scroll_capture_direction_from_delta_y(delta_y); + let inferred_direction = + Self::scroll_capture_direction_from_external_input_delta_y(delta_y); let input_age_ms = u64::try_from(through.saturating_duration_since(recorded_at).as_millis()) .unwrap_or(u64::MAX); @@ -201,7 +369,27 @@ impl OverlaySession { gesture_ended, through, ); - self.refresh_live_stream_stale_grace_for_external_input(seq); + let snapshot_after = self.scroll_capture_trace_snapshot_at(through); + if let Some(trace_recorder) = self.scroll_capture.trace_recorder.as_mut() { + trace_recorder.record_replayed_input(ScrollCaptureTraceInputRecord { + seq, + cursor_global: (global_x, global_y), + delta_y, + gesture_active, + gesture_ended, + recorded_age: through.saturating_duration_since(recorded_at), + applied_at: through, + snapshot_after, + }); + } + if !self.should_use_scroll_capture_worker_sampling() { + self.refresh_live_stream_stale_grace_for_external_input(seq); + } + if self.scroll_capture.active && !self.scroll_capture.paused { + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + self.request_redraw_scroll_preview_window(); + } tracing::info!( op = "scroll_capture.replayed_input_result", @@ -213,12 +401,180 @@ impl OverlaySession { paired_at_age_ms = self.scroll_capture_input_age_ms(), after_direction = ?self.scroll_capture.input_direction, after_gesture_active = self.scroll_capture.input_gesture_active, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, "Applied replayed external scroll input event to scroll-capture state." ); } } - #[cfg(any(not(target_os = "macos"), test))] + #[cfg(target_os = "macos")] + pub(super) fn note_scroll_capture_live_stream_frame_activity( + &mut self, + frame: &ScrollCaptureLiveFrame, + ) -> bool { + let fingerprint = scroll_capture_fingerprint(&frame.image); + let is_distinct = + self.scroll_capture.last_stream_frame_fingerprint.as_ref().map_or(true, |previous| { + scroll_capture_fingerprint_delta(previous, &fingerprint) > 0 + }); + + self.scroll_capture.last_stream_frame_fingerprint = Some(fingerprint); + if is_distinct { + self.scroll_capture.last_stream_event_at = Some(frame.captured_at); + self.scroll_capture.consecutive_identical_stream_frames = 0; + self.scroll_capture.last_duplicate_stream_refresh_at = None; + } else { + self.scroll_capture.consecutive_identical_stream_frames = + self.scroll_capture.consecutive_identical_stream_frames.saturating_add(1); + } + + is_distinct + } + + #[cfg(target_os = "macos")] + pub(super) fn maybe_schedule_duplicate_stream_refresh( + &mut self, + frame_seq: u64, + observation_at: Instant, + ) { + if self.scroll_capture.consecutive_identical_stream_frames + < SCROLL_CAPTURE_DUPLICATE_STREAM_STALL_THRESHOLD + { + return; + } + if !self.scroll_capture_has_fresh_downward_backlog_at(observation_at) { + return; + } + if self.scroll_capture.last_duplicate_stream_refresh_at.is_some_and(|last| { + observation_at.saturating_duration_since(last) + < SCROLL_CAPTURE_DUPLICATE_STREAM_REFRESH_INTERVAL + }) { + return; + } + + let Some(monitor) = self.scroll_capture.monitor else { + return; + }; + let Some(live_stream) = self.scroll_capture.live_stream.as_ref() else { + return; + }; + + let refresh_scheduled = + live_stream.refresh_monitor_nonblocking_if_stale(monitor, frame_seq, true); + + if !refresh_scheduled { + return; + } + + self.scroll_capture.last_duplicate_stream_refresh_at = Some(observation_at); + self.scroll_capture.pending_post_stall_burst_after_seq = Some(frame_seq); + tracing::info!( + op = "scroll_capture.duplicate_frame_refresh_scheduled", + frame_seq, + identical_streak = self.scroll_capture.consecutive_identical_stream_frames, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, + "Scheduled a forced live-stream refresh after repeated identical frames during fresh downward backlog." + ); + } + + #[cfg(target_os = "macos")] + fn push_scroll_capture_live_frame(&mut self, frame: ScrollCaptureLiveFrame) { + let backlog = &mut self.scroll_capture.live_stream_backlog; + if backlog.len() >= super::SCROLL_CAPTURE_STREAM_BACKLOG_MAX_FRAMES { + backlog.pop_front(); + } + backlog.push_back(frame); + } + + #[cfg(all(test, target_os = "macos"))] + pub(super) fn test_push_scroll_capture_live_frame(&mut self, frame: ScrollCaptureLiveFrame) { + self.push_scroll_capture_live_frame(frame); + } + + #[cfg(target_os = "macos")] + fn consume_scroll_capture_backlog(&mut self, max_frames: usize) { + let mut consumed = 0; + while consumed < max_frames { + let Some(frame) = self.scroll_capture.live_stream_backlog.pop_front() else { + break; + }; + self.drain_external_scroll_input_events_through(frame.captured_at); + let arm_time_gap_burst = + self.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(frame.captured_at); + if arm_time_gap_burst { + self.scroll_capture.pending_post_stall_burst_after_seq = + Some(frame.frame_seq.saturating_sub(1)); + tracing::info!( + op = "scroll_capture.post_stall_burst_search_armed_for_time_gap", + frame_seq = frame.frame_seq, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, + last_consumed_captured_at = ?self.scroll_capture.last_consumed_stream_frame_captured_at, + current_captured_at = ?frame.captured_at, + "Armed a burst registration window because the next live-stream frame arrived after a large capture-time gap while fresh downward backlog remained." + ); + } + let _is_distinct = self.note_scroll_capture_live_stream_frame_activity(&frame); + self.scroll_capture.last_stream_frame_seq = frame.frame_seq; + + let _ = self.handle_scroll_capture_frame( + frame.image, + ScrollCaptureFrameSource::LiveStream { frame_seq: frame.frame_seq }, + false, + frame.captured_at, + ); + self.scroll_capture.last_consumed_stream_frame_captured_at = Some(frame.captured_at); + self.maybe_schedule_duplicate_stream_refresh(frame.frame_seq, frame.captured_at); + + consumed += 1; + } + } + + #[cfg(all(test, target_os = "macos"))] + pub(super) fn test_consume_scroll_capture_backlog(&mut self, max_frames: usize) { + self.consume_scroll_capture_backlog(max_frames); + } + + #[cfg(target_os = "macos")] + pub(super) fn replay_recorded_live_stream_frame( + &mut self, + frame: RgbaImage, + frame_seq: u64, + observed_at: Instant, + allow_stale_input: bool, + ) -> Option> { + if self.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(observed_at) { + self.scroll_capture.pending_post_stall_burst_after_seq = + Some(frame_seq.saturating_sub(1)); + } + + let frame_for_activity = + ScrollCaptureLiveFrame { frame_seq, captured_at: observed_at, image: frame.clone() }; + let _ = self.note_scroll_capture_live_stream_frame_activity(&frame_for_activity); + self.scroll_capture.last_stream_frame_seq = frame_seq; + let outcome = self.handle_scroll_capture_frame( + frame, + ScrollCaptureFrameSource::LiveStream { frame_seq }, + allow_stale_input, + observed_at, + ); + self.scroll_capture.last_consumed_stream_frame_captured_at = Some(observed_at); + self.maybe_schedule_duplicate_stream_refresh(frame_seq, observed_at); + + outcome + } + + #[cfg(target_os = "macos")] + fn poll_scroll_stream_fallback_if_due(&mut self, now: Instant) { + let should_poll_stream_fallback = + self.scroll_capture.last_stream_event_at.is_none_or(|last| { + now.duration_since(last) >= SCROLL_CAPTURE_STREAM_EVENT_FALLBACK_POLL_INTERVAL + }); + + if should_poll_stream_fallback { + let _ = self.try_consume_scroll_stream_frame(); + } + } + pub(super) fn handle_captured_scroll_region( &mut self, monitor: MonitorRect, @@ -285,11 +641,29 @@ impl OverlaySession { #[cfg(target_os = "macos")] let allow_stale_input_for_request = self.allow_worker_frame_with_latched_request_input(request_id); + #[cfg(target_os = "macos")] + let request_input_was_superseded = self.worker_frame_request_input_was_superseded(request_id); #[cfg(not(target_os = "macos"))] let allow_stale_input_for_request = false; + #[cfg(not(target_os = "macos"))] + let request_input_was_superseded = false; + + if request_input_was_superseded { + tracing::info!( + op = "scroll_capture.worker_frame_dropped", + reason = "superseded_input_context", + request_id, + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + last_external_scroll_input_seq = self.scroll_capture.last_external_scroll_input_seq, + "Dropped worker-fed scroll-capture frame because newer external input superseded the request context." + ); + self.clear_scroll_capture_inflight_request(); + return; + } self.clear_scroll_capture_inflight_request(); - self.handle_scroll_capture_frame( + let _ = self.handle_scroll_capture_frame( image, ScrollCaptureFrameSource::Worker { request_id }, allow_stale_input_for_request, @@ -297,7 +671,6 @@ impl OverlaySession { ); } - #[cfg(any(not(target_os = "macos"), test))] pub(super) fn handle_missing_scroll_region( &mut self, monitor: MonitorRect, @@ -352,6 +725,11 @@ impl OverlaySession { } self.clear_scroll_capture_inflight_request(); + #[cfg(target_os = "macos")] + self.schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( + Instant::now(), + "worker_no_new_frame", + ); tracing::info!( op = "scroll_capture.worker_frame_unavailable", @@ -368,34 +746,25 @@ impl OverlaySession { source: ScrollCaptureFrameSource, allow_stale_input: bool, observation_at: Instant, - ) { + ) -> Option> { + let trace_frame = self.scroll_capture.trace_recorder.as_ref().map(|_| frame.clone()); + let preview_frame = frame.clone(); let frame_px = frame.dimensions(); - - if let Some(reason) = self.scroll_capture_observation_block_reason_at(observation_at) { - #[cfg(target_os = "macos")] - let allow_live_stream_stale_grace = !allow_stale_input - && reason == "stale_input" + let prior_block_reason = self.scroll_capture_observation_block_reason_at(observation_at); + #[cfg(target_os = "macos")] + let allow_stale_input = allow_stale_input + || (!allow_stale_input + && prior_block_reason == Some("stale_input") && matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) - && self.consume_live_stream_stale_grace_if_current(); - #[cfg(not(target_os = "macos"))] - let allow_live_stream_stale_grace = false; - - if (allow_stale_input || allow_live_stream_stale_grace) && reason == "stale_input" { - let Some(outcome) = - self.observe_scroll_capture_frame_with_gate(frame, true, observation_at) - else { - return; - }; - - self.handle_scroll_capture_frame_outcome(outcome, source, frame_px); - - return; - } + && self.consume_live_stream_stale_grace_if_current()); + #[cfg(not(target_os = "macos"))] + let allow_stale_input = allow_stale_input; + if let Some(reason) = prior_block_reason { let input_age_ms = self.scroll_capture_input_age_ms_at(observation_at); tracing::info!( - op = "scroll_capture.observation_blocked", + op = "scroll_capture.observation_prior_state", frame_source = source.as_str(), worker_request_id = ?source.worker_request_id(), reason, @@ -403,28 +772,106 @@ impl OverlaySession { input_direction = ?self.scroll_capture.input_direction, input_gesture_active = self.scroll_capture.input_gesture_active, input_age_ms = ?input_age_ms, - "Skipped scroll-capture frame observation because input was not currently usable." + allow_stale_input, + "Observed a scroll-capture frame while input metadata would previously have blocked observation." ); + } - return; + let allow_post_stall_burst_search = match source { + ScrollCaptureFrameSource::LiveStream { frame_seq } => self + .scroll_capture_should_allow_post_stall_burst_search_at(frame_seq, observation_at), + ScrollCaptureFrameSource::Worker { .. } => false, + }; + + if allow_post_stall_burst_search { + tracing::info!( + op = "scroll_capture.post_stall_burst_search_enabled", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + pending_after_seq = ?self.scroll_capture.pending_post_stall_burst_after_seq, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, + "Kept a burst registration window enabled after an explicit stalled-refresh episode while fresh downward backlog remained." + ); } - let Some(outcome) = self.observe_scroll_capture_frame_at(frame, observation_at) else { - return; + let worker_pairwise_path = + cfg!(target_os = "macos") && matches!(source, ScrollCaptureFrameSource::Worker { .. }); + let outcome = if worker_pairwise_path { + let Some(session) = self.scroll_capture.session.as_mut() else { + self.scroll_capture_set_error("Scroll capture session is unavailable."); + + return None; + }; + + session.observe_worker_pairwise_vision_frame(frame) + } else { + self.observe_scroll_capture_frame_with_gate( + frame, + false, + observation_at, + allow_post_stall_burst_search, + )? }; + if worker_pairwise_path { + if let Ok(outcome) = &outcome { + self.consume_scroll_capture_downward_motion_rows_for_outcome(outcome); + } + } + if matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) + && !allow_post_stall_burst_search + { + self.scroll_capture.pending_post_stall_burst_after_seq = None; + } - self.handle_scroll_capture_frame_outcome(outcome, source, frame_px); + self.scroll_capture.preview_latest_frame = Some(preview_frame); + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + self.request_redraw_scroll_preview_window(); + self.handle_scroll_capture_frame_outcome(&outcome, source, frame_px); + let snapshot_after = self.scroll_capture_trace_snapshot_at(observation_at); + + if let (Some(trace_recorder), Some(trace_frame)) = + (self.scroll_capture.trace_recorder.as_mut(), trace_frame.as_ref()) + { + trace_recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: trace_frame, + source, + allow_stale_input, + prior_block_reason, + observed_at: observation_at, + snapshot_after, + outcome: &outcome, + }); + } + + Some(outcome) } fn handle_scroll_capture_frame_outcome( &mut self, - outcome: Result, + outcome: &Result, source: ScrollCaptureFrameSource, frame_px: (u32, u32), ) { match outcome { Ok(ScrollObserveOutcome::NoChange) => { + tracing::info!( + op = "scroll_capture.frame_observed", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + outcome = "no_change", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + input_gesture_active = self.scroll_capture.input_gesture_active, + export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), + "Scroll-capture observed a frame but kept session state unchanged." + ); if let Some(request_id) = source.worker_request_id() { + #[cfg(target_os = "macos")] + self.schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( + Instant::now(), + "worker_no_change", + ); tracing::info!( op = "scroll_capture.worker_frame_processed", request_id, @@ -436,9 +883,21 @@ impl OverlaySession { } }, Ok(ScrollObserveOutcome::PreviewUpdated) => { + tracing::info!( + op = "scroll_capture.frame_observed", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + outcome = "preview_updated", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + input_gesture_active = self.scroll_capture.input_gesture_active, + export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), + preview_px = ?self.scroll_capture_preview_dimensions().map(|[w, h]| (w, h)), + "Scroll-capture observed a frame and advanced session sampling state without committing stitched growth." + ); if let Some(request_id) = source.worker_request_id() { tracing::info!( - op = "scroll_capture.worker_frame_processed", + op = "scroll_capture.worker_frame_processed", request_id, outcome = "preview_updated", frame_px = ?frame_px, @@ -465,25 +924,46 @@ impl OverlaySession { ); }, Ok(ScrollObserveOutcome::Committed { direction, growth_rows }) => { - let export_size = self - .scroll_capture - .session - .as_ref() - .map_or((0, 0), ScrollSession::export_dimensions); + self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + self.request_redraw_scroll_preview_window(); + + let telemetry = + self.scroll_capture.session.as_ref().map(ScrollSession::commit_telemetry); + let export_size = + telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.export_dimensions); + let preview_size = + telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.preview_dimensions); tracing::info!( op = "scroll_capture.committed", frame_source = source.as_str(), worker_request_id = ?source.worker_request_id(), direction = ?direction, - growth_rows, + growth_rows = *growth_rows, frame_px = ?frame_px, export_px = ?export_size, + preview_px = ?preview_size, + current_viewport_top_y = ?telemetry.as_ref().map(|telemetry| telemetry.current_viewport_top_y), + growth_commit_count = ?telemetry.as_ref().map(|telemetry| telemetry.growth_commit_count), + preview_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.preview_segment_count), + export_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.export_segment_count), + last_commit_decision_source = + ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_decision_source), + last_commit_detected_motion_rows = + ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_detected_motion_rows), + last_commit_effective_motion_rows_hint = ?telemetry + .as_ref() + .map(|telemetry| telemetry.last_commit_effective_motion_rows_hint), + last_preview_segment_height_px = + ?telemetry.as_ref().map(|telemetry| telemetry.last_preview_segment_height_px), + last_export_segment_height_px = + ?telemetry.as_ref().map(|telemetry| telemetry.last_export_segment_height_px), + preview_export_segments_aligned = + ?telemetry.as_ref().map(|telemetry| telemetry.preview_export_segments_aligned), "Scroll sample committed stitched growth." ); - - self.sync_scroll_preview_segments(); - self.request_redraw_scroll_preview_window(); }, Err(err) => { self.scroll_capture_set_error(format!("{err:#}")); @@ -499,7 +979,7 @@ impl OverlaySession { } } - #[cfg(all(target_os = "macos", test))] + #[cfg(target_os = "macos")] pub(super) fn allow_worker_frame_with_latched_request_input(&self, request_id: u64) -> bool { if self.scroll_capture.inflight_request_id != Some(request_id) { return false; @@ -512,11 +992,34 @@ impl OverlaySession { if !observation.was_observable { return false; } - if observation.external_input_seq != self.scroll_capture.last_external_scroll_input_seq { + + matches!( + (observation.input_direction, self.scroll_capture.input_direction), + (Some(request_direction), Some(current_direction)) + if request_direction == current_direction + ) + } + + #[cfg(target_os = "macos")] + fn worker_frame_request_input_was_superseded(&self, request_id: u64) -> bool { + if self.scroll_capture.inflight_request_id != Some(request_id) { return false; } - observation.input_direction == self.scroll_capture.input_direction + let Some(observation) = self.scroll_capture.inflight_request_observation else { + return false; + }; + if !observation.was_observable + || observation.external_input_seq == self.scroll_capture.last_external_scroll_input_seq + { + return false; + } + + matches!( + (observation.input_direction, self.scroll_capture.input_direction), + (Some(request_direction), Some(current_direction)) + if request_direction != current_direction + ) } #[cfg(target_os = "macos")] @@ -526,10 +1029,8 @@ impl OverlaySession { }; let grace_is_current = grace.external_input_seq == self.scroll_capture.last_external_scroll_input_seq; - let grace_is_compatible = self.scroll_capture.input_direction - == Some(grace.input_direction) - && !self.scroll_capture.input_gesture_active - && grace.input_direction == ScrollDirection::Down; + let grace_is_compatible = self.scroll_capture.input_direction.is_some() + && !self.scroll_capture.input_gesture_active; if !(grace_is_current && grace_is_compatible) { self.scroll_capture.live_stream_stale_grace = None; @@ -541,15 +1042,16 @@ impl OverlaySession { &mut self, external_input_seq: u64, ) { - self.scroll_capture.live_stream_stale_grace = - match (self.scroll_capture.input_direction, self.scroll_capture.input_gesture_active) { - (Some(ScrollDirection::Down), false) => Some(LiveStreamStaleGrace { - external_input_seq, - input_direction: ScrollDirection::Down, - remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, - }), - _ => None, - }; + self.scroll_capture.live_stream_stale_grace = match ( + self.scroll_capture.input_direction.is_some(), + self.scroll_capture.input_gesture_active, + ) { + (true, false) => Some(LiveStreamStaleGrace { + external_input_seq, + remaining_stale_frames: SCROLL_CAPTURE_LIVE_STREAM_STALE_GRACE_FRAMES, + }), + _ => None, + }; } #[cfg(target_os = "macos")] @@ -559,9 +1061,8 @@ impl OverlaySession { }; if grace.external_input_seq != self.scroll_capture.last_external_scroll_input_seq - || self.scroll_capture.input_direction != Some(grace.input_direction) + || self.scroll_capture.input_direction.is_none() || self.scroll_capture.input_gesture_active - || grace.input_direction != ScrollDirection::Down || grace.remaining_stale_frames == 0 { self.scroll_capture.live_stream_stale_grace = None; diff --git a/packages/rsnap-overlay/src/overlay/session_state.rs b/packages/rsnap-overlay/src/overlay/session_state.rs index 8d253f34..34a66ff3 100644 --- a/packages/rsnap-overlay/src/overlay/session_state.rs +++ b/packages/rsnap-overlay/src/overlay/session_state.rs @@ -6,7 +6,8 @@ use std::{ use crate::overlay::{ DeviceCursorPointSource, FrozenToolbarTool, GlobalPoint, LIVE_PRESENT_INTERVAL_MIN, MonitorRect, PhysicalPosition, Pos2, REDRAW_SUBSTEP_CONTRIBUTION_FLOOR, RectPoints, - SLOW_OP_WARN_INTERVAL, ScrollDirection, ScrollSession, Vec2, WindowId, + SLOW_OP_WARN_INTERVAL, ScrollCaptureTraceRecorder, ScrollDirection, ScrollSession, Vec2, + WindowId, }; #[cfg(target_os = "macos")] use crate::overlay::{ExternalScrollInputDrainReader, MacLiveFrameStream}; @@ -173,13 +174,17 @@ pub(super) struct ScrollCaptureState { pub(super) active: bool, pub(super) paused: bool, pub(super) monitor: Option, + pub(super) capture_rect_points: Option, pub(super) capture_rect_pixels: Option, pub(super) input_direction: Option, pub(super) input_direction_at: Option, pub(super) input_gesture_active: bool, + pub(super) downward_motion_rows_pending: f64, #[cfg(target_os = "macos")] pub(super) overlay_mouse_passthrough_active: bool, #[cfg(target_os = "macos")] + pub(super) overlay_mouse_passthrough_persistent: bool, + #[cfg(target_os = "macos")] pub(super) overlay_mouse_passthrough_until: Option, #[cfg(target_os = "macos")] pub(super) external_scroll_input_drain_reader: Option, @@ -190,32 +195,74 @@ pub(super) struct ScrollCaptureState { #[cfg(target_os = "macos")] pub(super) live_stream: Option, #[cfg(target_os = "macos")] + pub(super) live_stream_backlog: std::collections::VecDeque, + #[cfg(target_os = "macos")] pub(super) last_stream_frame_seq: u64, #[cfg(target_os = "macos")] + pub(super) last_stream_frame_fingerprint: Option>, + #[cfg(target_os = "macos")] + pub(super) consecutive_identical_stream_frames: u8, + #[cfg(target_os = "macos")] + pub(super) last_consumed_stream_frame_captured_at: Option, + #[cfg(target_os = "macos")] + pub(super) last_stream_event_at: Option, + #[cfg(target_os = "macos")] + pub(super) last_stream_poll_at: Option, + #[cfg(target_os = "macos")] + pub(super) last_duplicate_stream_refresh_at: Option, + #[cfg(target_os = "macos")] + pub(super) pending_post_stall_burst_after_seq: Option, + #[cfg(target_os = "macos")] pub(super) live_stream_stale_grace: Option, - #[cfg(not(target_os = "macos"))] pub(super) next_sample_at: Option, - #[cfg(not(target_os = "macos"))] pub(super) next_request_id: u64, pub(super) inflight_request_id: Option, #[cfg(target_os = "macos")] pub(super) inflight_request_observation: Option, + #[cfg(all(test, target_os = "macos"))] + pub(super) force_worker_sampling_in_tests: bool, pub(super) session: Option, + pub(super) preview_committed_image: Option, + pub(super) preview_latest_frame: Option, + pub(super) preview_display_image: Option, + pub(super) retained_overlay_preview_image: Option, + pub(super) retained_overlay_preview_motion_rows_hint: Option, + pub(super) last_overlay_preview_motion_rows_hint: Option, + pub(super) last_overlay_preview_provisional_motion_rows_hint: Option, + pub(super) last_overlay_preview_existing_candidate_height: Option, + pub(super) last_overlay_preview_existing_candidate_motion_rows_hint: Option, + pub(super) last_overlay_preview_ledger_candidate_height: Option, + pub(super) last_overlay_preview_ledger_candidate_motion_rows_hint: Option, + pub(super) last_overlay_preview_retained_candidate_height: Option, + pub(super) last_overlay_preview_retained_candidate_motion_rows_hint: Option, + pub(super) last_overlay_preview_retained_hint_matches_motion_rows: bool, + pub(super) last_overlay_preview_fresh_latest_frame_can_drive: bool, + pub(super) last_overlay_preview_strong_unresolved_registration: bool, + pub(super) last_overlay_preview_latest_frame_present: bool, + pub(super) last_overlay_preview_used_provisional: bool, + pub(super) trace_recorder: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug)] +pub(super) struct ScrollCaptureLiveFrame { + pub(super) frame_seq: u64, + pub(super) captured_at: Instant, + pub(super) image: image::RgbaImage, } #[cfg(target_os = "macos")] #[derive(Clone, Copy, Debug, Default, PartialEq)] pub(super) struct InflightScrollCaptureObservation { - pub(super) input_direction: Option, pub(super) was_observable: bool, pub(super) external_input_seq: u64, + pub(super) input_direction: Option, } #[cfg(target_os = "macos")] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) struct LiveStreamStaleGrace { pub(super) external_input_seq: u64, - pub(super) input_direction: ScrollDirection, pub(super) remaining_stale_frames: u8, } diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs new file mode 100644 index 00000000..45a3c377 --- /dev/null +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -0,0 +1,739 @@ +use std::{ + env, fs, + path::{Path, PathBuf}, + process, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use color_eyre::eyre::{Result, WrapErr}; +use directories::ProjectDirs; +use image::RgbaImage; +use serde::{Deserialize, Serialize}; + +use super::{MonitorRect, RectPoints, ScrollCaptureFrameSource}; +use crate::{ + png, + scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}, +}; + +const SCROLL_CAPTURE_TRACE_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE"; +const SCROLL_CAPTURE_TRACE_DIR_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE_DIR"; +const SCROLL_CAPTURE_TRACE_SCHEMA: &str = "scroll_capture_live_trace/1"; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureLiveTraceManifest { + pub(crate) schema: String, + pub(crate) trace_id: String, + pub(crate) started_unix_ms: u64, + pub(crate) preview_width_px: u32, + pub(crate) monitor: ScrollCaptureTraceMonitor, + pub(crate) capture_rect_pixels: ScrollCaptureTraceRect, + pub(crate) base_frame_path: String, + pub(crate) entries: Vec, + pub(crate) final_preview_path: Option, + pub(crate) final_export_path: Option, + pub(crate) final_snapshot: Option, + pub(crate) final_error: Option, + pub(crate) finalized: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureTraceMonitor { + pub(crate) id: u32, + pub(crate) origin_x: i32, + pub(crate) origin_y: i32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) scale_factor_x1000: u32, +} + +impl From for ScrollCaptureTraceMonitor { + fn from(value: MonitorRect) -> Self { + Self { + id: value.id, + origin_x: value.origin.x, + origin_y: value.origin.y, + width: value.width, + height: value.height, + scale_factor_x1000: value.scale_factor_x1000, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureTraceRect { + pub(crate) x: u32, + pub(crate) y: u32, + pub(crate) width: u32, + pub(crate) height: u32, +} + +impl From for ScrollCaptureTraceRect { + fn from(value: RectPoints) -> Self { + Self { x: value.x, y: value.y, width: value.width, height: value.height } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "entry_type", rename_all = "snake_case")] +pub(crate) enum ScrollCaptureLiveTraceEntry { + Input(ScrollCaptureTraceInputEntry), + Frame(ScrollCaptureTraceFrameEntry), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureTraceInputEntry { + pub(crate) applied_at_ms: u64, + pub(crate) seq: u64, + pub(crate) cursor_global_x: f64, + pub(crate) cursor_global_y: f64, + pub(crate) delta_y: f64, + pub(crate) gesture_active: bool, + pub(crate) gesture_ended: bool, + pub(crate) recorded_age_ms: u64, + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureTraceFrameEntry { + pub(crate) observed_at_ms: u64, + pub(crate) allow_stale_input: bool, + pub(crate) prior_block_reason: Option, + pub(crate) frame_path: String, + pub(crate) frame_source: ScrollCaptureTraceFrameSource, + pub(crate) frame_dimensions: [u32; 2], + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, + pub(crate) outcome: ScrollCaptureTraceRecordedOutcome, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ScrollCaptureTraceFrameSource { + Worker { request_id: u64 }, + LiveStream { frame_seq: u64 }, +} + +impl From for ScrollCaptureTraceFrameSource { + fn from(value: ScrollCaptureFrameSource) -> Self { + match value { + ScrollCaptureFrameSource::Worker { request_id } => Self::Worker { request_id }, + #[cfg(target_os = "macos")] + ScrollCaptureFrameSource::LiveStream { frame_seq } => Self::LiveStream { frame_seq }, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ScrollCaptureTraceDirection { + Up, + Down, +} + +impl From for ScrollCaptureTraceDirection { + fn from(value: ScrollDirection) -> Self { + match value { + ScrollDirection::Up => Self::Up, + ScrollDirection::Down => Self::Down, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum ScrollCaptureTraceRecordedOutcome { + NoChange, + PreviewUpdated, + UnsupportedDirection { direction: ScrollCaptureTraceDirection }, + Committed { direction: ScrollCaptureTraceDirection, growth_rows: u32 }, + Error { message: String }, +} + +impl From for ScrollCaptureTraceRecordedOutcome { + fn from(value: ScrollObserveOutcome) -> Self { + match value { + ScrollObserveOutcome::NoChange => Self::NoChange, + ScrollObserveOutcome::PreviewUpdated => Self::PreviewUpdated, + ScrollObserveOutcome::UnsupportedDirection { direction } => { + Self::UnsupportedDirection { direction: direction.into() } + }, + ScrollObserveOutcome::Committed { direction, growth_rows } => { + Self::Committed { direction: direction.into(), growth_rows } + }, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ScrollCaptureTraceSessionSnapshot { + pub(crate) input_direction: Option, + pub(crate) input_gesture_active: bool, + pub(crate) downward_motion_rows_pending: f64, + pub(crate) input_age_ms: Option, + pub(crate) current_viewport_top_y: Option, + pub(crate) export_dimensions: Option<[u32; 2]>, + pub(crate) preview_dimensions: Option<[u32; 2]>, + pub(crate) growth_commit_count: Option, + pub(crate) preview_segment_count: Option, + pub(crate) export_segment_count: Option, + pub(crate) preview_export_segments_aligned: Option, + pub(crate) last_commit_decision_source: Option, + pub(crate) last_commit_detected_motion_rows: Option, + pub(crate) last_commit_effective_motion_rows_hint: Option, + pub(crate) last_preview_segment_height_px: Option, + pub(crate) last_export_segment_height_px: Option, +} + +impl ScrollCaptureTraceSessionSnapshot { + pub(crate) fn capture( + session: Option<&ScrollSession>, + preview_dimensions: Option<[u32; 2]>, + input_direction: Option, + input_gesture_active: bool, + downward_motion_rows_pending: f64, + input_age_ms: Option, + ) -> Self { + let telemetry = session.map(ScrollSession::commit_telemetry); + + Self { + input_direction: input_direction.map(Into::into), + input_gesture_active, + downward_motion_rows_pending, + input_age_ms, + current_viewport_top_y: session.map(ScrollSession::current_viewport_top_y), + export_dimensions: session.map(ScrollSession::export_dimensions).map(|(w, h)| [w, h]), + preview_dimensions, + growth_commit_count: telemetry.as_ref().map(|value| value.growth_commit_count), + preview_segment_count: telemetry.as_ref().map(|value| value.preview_segment_count), + export_segment_count: telemetry.as_ref().map(|value| value.export_segment_count), + preview_export_segments_aligned: telemetry + .as_ref() + .map(|value| value.preview_export_segments_aligned), + last_commit_decision_source: telemetry + .as_ref() + .and_then(|value| value.last_commit_decision_source) + .map(str::to_owned), + last_commit_detected_motion_rows: telemetry + .as_ref() + .and_then(|value| value.last_commit_detected_motion_rows), + last_commit_effective_motion_rows_hint: telemetry + .as_ref() + .and_then(|value| value.last_commit_effective_motion_rows_hint), + last_preview_segment_height_px: telemetry + .as_ref() + .and_then(|value| value.last_preview_segment_height_px), + last_export_segment_height_px: telemetry + .as_ref() + .and_then(|value| value.last_export_segment_height_px), + } + } +} + +pub(crate) struct ScrollCaptureTraceRecorder { + trace_dir: PathBuf, + manifest_path: PathBuf, + started_at: Instant, + next_frame_index: u64, + manifest: ScrollCaptureLiveTraceManifest, +} + +pub(crate) struct ScrollCaptureTraceInputRecord { + pub(crate) seq: u64, + pub(crate) cursor_global: (f64, f64), + pub(crate) delta_y: f64, + pub(crate) gesture_active: bool, + pub(crate) gesture_ended: bool, + pub(crate) recorded_age: Duration, + pub(crate) applied_at: Instant, + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, +} + +pub(crate) struct ScrollCaptureTraceFrameRecord<'a> { + pub(crate) frame: &'a RgbaImage, + pub(crate) source: ScrollCaptureFrameSource, + pub(crate) allow_stale_input: bool, + pub(crate) prior_block_reason: Option<&'static str>, + pub(crate) observed_at: Instant, + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, + pub(crate) outcome: &'a Result, +} + +impl ScrollCaptureTraceRecorder { + pub(crate) fn from_env( + monitor: MonitorRect, + capture_rect_pixels: RectPoints, + preview_width_px: u32, + base_frame: &RgbaImage, + ) -> Option { + let trace_root = resolve_trace_root_dir()?; + + match Self::new_for_root_dir( + trace_root, + monitor, + capture_rect_pixels, + preview_width_px, + base_frame, + ) { + Ok(recorder) => Some(recorder), + Err(err) => { + tracing::warn!( + op = "scroll_capture.trace_init_failed", + error = %err, + "Failed to initialize scroll-capture live trace recorder." + ); + + None + }, + } + } + + pub(crate) fn record_replayed_input(&mut self, record: ScrollCaptureTraceInputRecord) { + self.manifest.entries.push(ScrollCaptureLiveTraceEntry::Input( + ScrollCaptureTraceInputEntry { + applied_at_ms: self.relative_ms(record.applied_at), + seq: record.seq, + cursor_global_x: record.cursor_global.0, + cursor_global_y: record.cursor_global.1, + delta_y: record.delta_y, + gesture_active: record.gesture_active, + gesture_ended: record.gesture_ended, + recorded_age_ms: duration_to_ms(record.recorded_age), + snapshot_after: record.snapshot_after, + }, + )); + self.flush_manifest_best_effort("record_input"); + } + + pub(crate) fn record_frame_observation(&mut self, record: ScrollCaptureTraceFrameRecord<'_>) { + let frame_index = self.next_frame_index; + self.next_frame_index = self.next_frame_index.saturating_add(1); + let frame_path = format!("frames/frame-{frame_index:06}.png"); + + if let Err(err) = self.write_frame(record.frame, &frame_path) { + tracing::warn!( + op = "scroll_capture.trace_write_frame_failed", + error = %err, + frame_index, + "Failed to persist scroll-capture trace frame." + ); + } + + let outcome = match record.outcome { + Ok(value) => ScrollCaptureTraceRecordedOutcome::from(*value), + Err(err) => ScrollCaptureTraceRecordedOutcome::Error { message: format!("{err:#}") }, + }; + + self.manifest.entries.push(ScrollCaptureLiveTraceEntry::Frame( + ScrollCaptureTraceFrameEntry { + observed_at_ms: self.relative_ms(record.observed_at), + allow_stale_input: record.allow_stale_input, + prior_block_reason: record.prior_block_reason.map(str::to_owned), + frame_path, + frame_source: record.source.into(), + frame_dimensions: [record.frame.width(), record.frame.height()], + snapshot_after: record.snapshot_after, + outcome, + }, + )); + self.flush_manifest_best_effort("record_frame"); + } + + pub(crate) fn record_error(&mut self, message: &str) { + self.manifest.final_error = Some(message.to_owned()); + self.flush_manifest_best_effort("record_error"); + } + + pub(crate) fn finalize_session( + &mut self, + session: &ScrollSession, + final_preview_image: &RgbaImage, + final_snapshot: ScrollCaptureTraceSessionSnapshot, + ) { + let final_preview_path = String::from("frames/final-preview.png"); + let final_export_path = String::from("frames/final-export.png"); + + if let Err(err) = self.write_frame(final_preview_image, &final_preview_path) { + tracing::warn!( + op = "scroll_capture.trace_write_final_preview_failed", + error = %err, + manifest_path = %self.manifest_path.display(), + "Failed to persist final scroll-capture preview trace frame." + ); + } else { + self.manifest.final_preview_path = Some(final_preview_path); + } + + if let Err(err) = self.write_frame(session.export_image(), &final_export_path) { + tracing::warn!( + op = "scroll_capture.trace_write_final_export_failed", + error = %err, + manifest_path = %self.manifest_path.display(), + "Failed to persist final scroll-capture export trace frame." + ); + } else { + self.manifest.final_export_path = Some(final_export_path); + } + + self.manifest.final_snapshot = Some(final_snapshot); + self.flush_manifest_best_effort("finalize_session"); + } + + pub(crate) fn manifest_path(&self) -> &Path { + &self.manifest_path + } + + pub(crate) fn new_for_root_dir( + trace_root: PathBuf, + monitor: MonitorRect, + capture_rect_pixels: RectPoints, + preview_width_px: u32, + base_frame: &RgbaImage, + ) -> Result { + let started_at = Instant::now(); + let started_unix_ms = now_unix_ms()?; + let trace_id = format!("scroll-capture-{}-pid{}", started_unix_ms, process::id()); + let trace_dir = trace_root.join(&trace_id); + let frames_dir = trace_dir.join("frames"); + let manifest_path = trace_dir.join("manifest.json"); + + fs::create_dir_all(&frames_dir).wrap_err_with(|| { + format!("failed to create trace directory {}", trace_dir.display()) + })?; + + let manifest = ScrollCaptureLiveTraceManifest { + schema: SCROLL_CAPTURE_TRACE_SCHEMA.to_owned(), + trace_id, + started_unix_ms, + preview_width_px, + monitor: monitor.into(), + capture_rect_pixels: capture_rect_pixels.into(), + base_frame_path: "frames/base.png".to_owned(), + entries: Vec::new(), + final_preview_path: None, + final_export_path: None, + final_snapshot: None, + final_error: None, + finalized: false, + }; + let recorder = Self { trace_dir, manifest_path, started_at, next_frame_index: 0, manifest }; + + recorder.write_frame(base_frame, &recorder.manifest.base_frame_path)?; + recorder.flush_manifest_best_effort("init"); + + Ok(recorder) + } + + fn relative_ms(&self, at: Instant) -> u64 { + duration_to_ms(at.saturating_duration_since(self.started_at)) + } + + fn write_frame(&self, frame: &RgbaImage, relative_path: &str) -> Result<()> { + let target_path = self.trace_dir.join(relative_path); + let png_bytes = png::rgba_image_to_png_bytes(frame) + .wrap_err("failed to encode scroll-capture trace frame")?; + + fs::write(&target_path, png_bytes) + .wrap_err_with(|| format!("failed to write trace frame {}", target_path.display())) + } + + fn flush_manifest_best_effort(&self, op: &'static str) { + if let Err(err) = self.flush_manifest() { + tracing::warn!( + op = "scroll_capture.trace_flush_failed", + stage = op, + error = %err, + manifest_path = %self.manifest_path.display(), + "Failed to flush scroll-capture trace manifest." + ); + } + } + + fn flush_manifest(&self) -> Result<()> { + let bytes = serde_json::to_vec_pretty(&self.manifest) + .wrap_err("failed to serialize scroll-capture trace manifest")?; + let tmp_path = self.manifest_path.with_extension("json.tmp"); + + fs::write(&tmp_path, bytes).wrap_err_with(|| { + format!("failed to write temporary trace manifest {}", tmp_path.display()) + })?; + fs::rename(&tmp_path, &self.manifest_path).wrap_err_with(|| { + format!( + "failed to publish scroll-capture trace manifest {}", + self.manifest_path.display() + ) + }) + } +} + +impl Drop for ScrollCaptureTraceRecorder { + fn drop(&mut self) { + self.manifest.finalized = true; + self.flush_manifest_best_effort("drop"); + } +} + +#[derive(Clone, Debug)] +pub(crate) struct LoadedScrollCaptureLiveTrace { + pub(crate) manifest_path: PathBuf, + pub(crate) manifest: ScrollCaptureLiveTraceManifest, + pub(crate) base_frame: RgbaImage, +} + +impl LoadedScrollCaptureLiveTrace { + pub(crate) fn load(manifest_path: impl AsRef) -> Result { + let manifest_path = manifest_path.as_ref().to_path_buf(); + let manifest_bytes = fs::read(&manifest_path).wrap_err_with(|| { + format!("failed to read scroll-capture trace manifest {}", manifest_path.display()) + })?; + let manifest: ScrollCaptureLiveTraceManifest = serde_json::from_slice(&manifest_bytes) + .wrap_err("failed to decode scroll-capture trace manifest")?; + let base_dir = manifest_path.parent().ok_or_else(|| { + color_eyre::eyre::eyre!( + "trace manifest path {} has no parent directory", + manifest_path.display() + ) + })?; + let base_frame_path = base_dir.join(&manifest.base_frame_path); + let base_frame = image::open(&base_frame_path) + .wrap_err_with(|| { + format!( + "failed to open scroll-capture trace base frame {}", + base_frame_path.display() + ) + })? + .into_rgba8(); + + Ok(Self { manifest_path, manifest, base_frame }) + } + + pub(crate) fn base_dir(&self) -> &Path { + self.manifest_path.parent().expect("trace manifest path should have a parent directory") + } + + pub(crate) fn resolve_frame_path(&self, relative_path: &str) -> PathBuf { + self.base_dir().join(relative_path) + } +} + +fn resolve_trace_root_dir() -> Option { + let override_dir = env::var_os(SCROLL_CAPTURE_TRACE_DIR_ENV).and_then(|value| { + let trimmed = value.to_string_lossy().trim().to_owned(); + if trimmed.is_empty() { None } else { Some(PathBuf::from(trimmed)) } + }); + if let Some(dir) = override_dir { + return Some(dir); + } + + let enabled = env::var_os(SCROLL_CAPTURE_TRACE_ENV) + .map(|value| parse_truthy_flag(&value.to_string_lossy())) + .unwrap_or(false); + + if !enabled { + return None; + } + + ProjectDirs::from("ink", "hack", "rsnap") + .map(|dirs| dirs.data_dir().join("scroll-capture-traces")) +} + +fn parse_truthy_flag(value: &str) -> bool { + let normalized = value.trim().to_ascii_lowercase(); + + !matches!(normalized.as_str(), "" | "0" | "false" | "no" | "off") +} + +fn duration_to_ms(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +fn now_unix_ms() -> Result { + Ok(u64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .wrap_err("system clock is before unix epoch")? + .as_millis(), + ) + .unwrap_or(u64::MAX)) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + + use super::*; + use crate::GlobalPoint; + use crate::overlay::{OverlaySession, ScrollCaptureFrameSource}; + + static TRACE_TEST_ROOT_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn test_monitor() -> MonitorRect { + MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + } + } + + fn test_rect() -> RectPoints { + RectPoints::new(100, 120, 3, 5) + } + + fn make_window(rows: &[[u8; 4]], start: usize) -> RgbaImage { + let mut image = RgbaImage::new(3, 5); + + for (y, row) in rows[start..start + 5].iter().enumerate() { + for x in 0..3 { + image.put_pixel(x, y as u32, image::Rgba(*row)); + } + } + + image + } + + fn temp_trace_root() -> PathBuf { + let counter = TRACE_TEST_ROOT_COUNTER.fetch_add(1, Ordering::Relaxed); + let root = std::env::temp_dir().join(format!( + "rsnap-scroll-trace-test-{}-{}-{}", + now_unix_ms().unwrap_or(0), + process::id(), + counter + )); + let _ = fs::remove_dir_all(&root); + + root + } + + #[test] + fn trace_recorder_round_trips_manifest_and_frames() { + let rows = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + ]; + let base_frame = make_window(&rows, 0); + let next_frame = make_window(&rows, 1); + let root = temp_trace_root(); + let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( + root, + test_monitor(), + test_rect(), + 320, + &base_frame, + ) + .unwrap(); + let start = Instant::now(); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(test_monitor()); + session.scroll_capture.capture_rect_pixels = Some(test_rect()); + session.scroll_capture.session = Some(ScrollSession::new(base_frame.clone(), 320).unwrap()); + + recorder.record_replayed_input(ScrollCaptureTraceInputRecord { + seq: 1, + cursor_global: (150.0, 160.0), + delta_y: 4.0, + gesture_active: true, + gesture_ended: false, + recorded_age: Duration::from_millis(3), + applied_at: start + Duration::from_millis(10), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + Some(ScrollDirection::Down), + true, + 4.0, + Some(3), + ), + }); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: &next_frame, + source: ScrollCaptureFrameSource::LiveStream { frame_seq: 7 }, + allow_stale_input: false, + prior_block_reason: None, + observed_at: start + Duration::from_millis(20), + snapshot_after: ScrollCaptureTraceSessionSnapshot::capture( + session.scroll_capture.session.as_ref(), + session + .scroll_capture + .session + .as_ref() + .map(ScrollSession::preview_display_image) + .map(|image| [image.width(), image.height()]), + Some(ScrollDirection::Down), + true, + 4.0, + Some(1), + ), + outcome: &Ok(ScrollObserveOutcome::PreviewUpdated), + }); + let manifest_path = recorder.manifest_path().to_path_buf(); + + drop(recorder); + + let loaded = LoadedScrollCaptureLiveTrace::load(&manifest_path).unwrap(); + + assert_eq!(loaded.manifest.schema, SCROLL_CAPTURE_TRACE_SCHEMA); + assert_eq!(loaded.manifest.entries.len(), 2); + assert!(loaded.manifest.finalized); + assert_eq!(loaded.base_frame.dimensions(), base_frame.dimensions()); + assert!(loaded.resolve_frame_path("frames/frame-000000.png").exists()); + } + + #[test] + fn trace_recorder_persists_final_preview_export_artifacts() { + let rows = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + ]; + let base_frame = make_window(&rows, 0); + let root = temp_trace_root(); + let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( + root, + test_monitor(), + test_rect(), + 320, + &base_frame, + ) + .unwrap(); + let manifest_path = recorder.manifest_path().to_path_buf(); + let session = ScrollSession::new(base_frame.clone(), 320).unwrap(); + let final_snapshot = ScrollCaptureTraceSessionSnapshot::capture( + Some(&session), + Some({ + let image = session.preview_display_image(); + [image.width(), image.height()] + }), + Some(ScrollDirection::Down), + false, + 0.0, + Some(0), + ); + + let final_preview_image = session.preview_display_image(); + recorder.finalize_session(&session, &final_preview_image, final_snapshot); + drop(recorder); + + let loaded = LoadedScrollCaptureLiveTrace::load(&manifest_path).unwrap(); + + assert_eq!(loaded.manifest.final_preview_path.as_deref(), Some("frames/final-preview.png")); + assert_eq!(loaded.manifest.final_export_path.as_deref(), Some("frames/final-export.png")); + assert!(loaded.resolve_frame_path("frames/final-preview.png").exists()); + assert!(loaded.resolve_frame_path("frames/final-export.png").exists()); + assert!(loaded.manifest.final_snapshot.is_some()); + } +} diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 19122b4d..671325aa 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -263,12 +263,50 @@ use image::{ RgbaImage, imageops::{self, FilterType}, }; +#[cfg(target_os = "macos")] +use objc2::{AnyThread, runtime::AnyObject}; +#[cfg(target_os = "macos")] +use objc2_core_foundation::CFData; +#[cfg(target_os = "macos")] +use objc2_core_graphics::{ + CGBitmapInfo, CGColorRenderingIntent, CGColorSpace, CGDataProvider, CGImage, CGImageAlphaInfo, + CGImageByteOrderInfo, +}; +#[cfg(target_os = "macos")] +use objc2_foundation::{NSArray, NSDictionary}; +#[cfg(target_os = "macos")] +use objc2_vision::{VNImageOption, VNImageRequestHandler, VNTranslationalImageRegistrationRequest}; const FINGERPRINT_GRID_COLUMNS: u32 = 12; const FINGERPRINT_GRID_ROWS: u32 = 16; -const DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 12; -const INITIAL_DOWNWARD_MAX_MOTION_ROWS: u32 = 192; -const MOTION_SEARCH_BAND_ROWS: u32 = 96; +const DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 48; +const DOWNWARD_KEYFRAME_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 4; +const DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS: u32 = 24; +const LOCAL_DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 4; +const LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS: u32 = 12; +const DOWNWARD_REGISTRATION_AMBIGUOUS_GAP_ROWS: u32 = 24; +const DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS: u32 = 4; +const DOWNWARD_REGISTRATION_MIN_OVERLAP_DIVISOR: u32 = 3; +const DOWNWARD_KEYFRAME_SEARCH_LIMIT: usize = 4; +const DOWNWARD_KEYFRAME_MIN_OVERLAP_DIVISOR: u32 = 5; +const INITIAL_DOWNWARD_MAX_MOTION_ROWS: u32 = 256; +pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 24; +pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS: u32 = 12; +const PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS: u32 = 4; +const EXTREME_TRANSIENT_PREVIEW_LOCAL_TAIL_MULTIPLIER: u32 = 12; +const REPEATED_PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 4; +const TINY_OBSERVED_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 2; +const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 1; +const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MIN_LAST_HINT_ROWS: u32 = 7; +pub(crate) const UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS: u32 = 8; +const BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS: u32 = 40; +const DOWNWARD_COMMITTED_KEYFRAME_LOCAL_OVERRUN_MAX_ROWS: u32 = 24; +const FALLBACK_DOWNWARD_GROWTH_MIN_ROWS: u32 = 8; +const FALLBACK_DOWNWARD_GROWTH_MAX_ROWS: u32 = 16; +const TRANSIENT_MOTION_HINT_MAX_MULTIPLIER: u32 = 3; +const TRANSIENT_MOTION_HINT_MIN_CAP_ROWS: u32 = 12; +const WORKER_PAIRWISE_CORROBORATION_MIN_ROWS: u32 = 32; +const WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS: u32 = 24; const DIRECTION_WARNING_MARGIN_X100: u32 = 90; const RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100: u32 = 320; const INFORMATIVE_SPAN_ROW_SAMPLES: u32 = 24; @@ -347,6 +385,7 @@ impl ScrollFrameFingerprint { } } +#[cfg(test)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct OverlapMatch { pub(crate) rows: u32, @@ -354,6 +393,43 @@ pub(crate) struct OverlapMatch { pub(crate) mean_abs_diff_x100: u32, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ScrollCommitTelemetry { + pub(crate) current_viewport_top_y: i32, + pub(crate) preview_dimensions: (u32, u32), + pub(crate) export_dimensions: (u32, u32), + pub(crate) growth_commit_count: usize, + pub(crate) preview_segment_count: usize, + pub(crate) export_segment_count: usize, + pub(crate) preview_export_segments_aligned: bool, + pub(crate) last_commit_decision_source: Option<&'static str>, + pub(crate) last_commit_detected_motion_rows: Option, + pub(crate) last_commit_effective_motion_rows_hint: Option, + pub(crate) last_block_reason: Option<&'static str>, + pub(crate) last_downward_sample_registration_result: Option<&'static str>, + pub(crate) last_downward_sample_registration_source: Option<&'static str>, + pub(crate) last_downward_sample_registration_motion_rows: Option, + pub(crate) last_downward_sample_registration_provisional_viewport_top_y: Option, + pub(crate) observed_sample_registration_result: Option<&'static str>, + pub(crate) observed_sample_registration_reason: Option<&'static str>, + pub(crate) observed_sample_registration_motion_rows: Option, + pub(crate) observed_sample_registration_mean_abs_diff_x100: Option, + pub(crate) preview_only_local_registration_result: Option<&'static str>, + pub(crate) preview_only_local_registration_reason: Option<&'static str>, + pub(crate) preview_only_local_registration_motion_rows: Option, + pub(crate) preview_only_local_registration_mean_abs_diff_x100: Option, + pub(crate) last_downward_viewport_candidate_count: Option, + pub(crate) last_downward_viewport_candidates_before_prune: Option, + pub(crate) last_downward_viewport_candidates_after_prune: Option, + pub(crate) sample_eval_last_motion_rows_hint: Option, + pub(crate) sample_eval_transient_motion_rows_hint: Option, + pub(crate) sample_eval_effective_motion_rows_hint: Option, + pub(crate) sample_eval_transient_burst_search_enabled: bool, + pub(crate) preview_only_local_viewport_top_y: Option, + pub(crate) last_preview_segment_height_px: Option, + pub(crate) last_export_segment_height_px: Option, +} + #[derive(Clone, Copy, Debug)] pub(crate) struct OverlapSearchConfig { pub(crate) min_overlap_rows: u32, @@ -366,12 +442,27 @@ impl Default for OverlapSearchConfig { Self { min_overlap_rows: 24, max_column_samples: 32, - max_row_samples: 8, + max_row_samples: 16, max_mean_abs_diff_x100: 850, } } } +fn worker_pairwise_overlap_search_config() -> OverlapSearchConfig { + OverlapSearchConfig { + min_overlap_rows: 24, + max_column_samples: 96, + max_row_samples: 96, + max_mean_abs_diff_x100: 850, + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreviewOnlyDownwardLocalSample { + frame: RgbaImage, + viewport_top_y: i32, +} + #[derive(Clone, Debug)] pub(crate) struct ScrollSession { anchor_frame: RgbaImage, @@ -382,10 +473,49 @@ pub(crate) struct ScrollSession { bottom_preview_segments: Vec, growth_history: Vec, last_committed_frame: RgbaImage, + worker_pairwise_previous_frame: RgbaImage, last_sample_frame: RgbaImage, last_sample_fingerprint: Option>, + last_downward_observed_frame: RgbaImage, + last_downward_observed_fingerprint: Option>, + last_preview_only_downward_local_sample: Option, + seeded_preview_only_local_after_observed_burst_commit: bool, + pending_unresolved_burst_registered_growth_viewport_top_y: Option, + last_blocked_preview_only_local_candidate: Option, + pending_suppressed_huge_preview_only_local_followup: Option, + pending_suppressed_huge_preview_only_local_followup_remaining_blocks: u8, + pending_extreme_preview_only_local_tail_followup: Option, + pending_extreme_preview_only_local_tail_followup_remaining_blocks: u8, last_unconfirmed_upward_fingerprint: Option>, last_motion_rows_hint: Option, + transient_motion_rows_hint: Option, + transient_burst_search_enabled: bool, + last_downward_sample_registration_result: Option<&'static str>, + last_downward_sample_registration_source: Option<&'static str>, + last_downward_sample_registration_motion_rows: Option, + last_downward_sample_registration_provisional_viewport_top_y: Option, + last_observed_sample_registration_result: Option<&'static str>, + last_observed_sample_registration_reason: Option<&'static str>, + last_observed_sample_registration_motion_rows: Option, + last_observed_sample_registration_mean_abs_diff_x100: Option, + last_preview_only_local_registration_result: Option<&'static str>, + last_preview_only_local_registration_reason: Option<&'static str>, + last_preview_only_local_registration_motion_rows: Option, + last_preview_only_local_registration_mean_abs_diff_x100: Option, + last_downward_viewport_candidate_count: Option, + last_downward_viewport_candidates_before_prune: Option, + last_downward_viewport_candidates_after_prune: Option, + blocked_underconsumed_observed_recovery_in_burst: bool, + blocked_lagging_exactly_corroborated_preview_local_tail_in_burst: bool, + blocked_followup_after_suppressed_huge_preview_local_jump: bool, + blocked_followup_after_extreme_preview_local_tail: bool, + blocked_far_committed_only_recovery_after_corroborated_huge_local_jump: bool, + consecutive_transient_burst_missing_downward_candidate_frames: u32, + last_block_reason: Option<&'static str>, + last_sample_eval_last_motion_rows_hint: Option, + last_sample_eval_transient_motion_rows_hint: Option, + last_sample_eval_effective_motion_rows_hint: Option, + last_sample_eval_transient_burst_search_enabled: bool, current_viewport_top_y: i32, observed_viewport_top_y: i32, resume_frontier_top_y: Option, @@ -406,10 +536,49 @@ impl ScrollSession { bottom_preview_segments: Vec::new(), growth_history: Vec::new(), last_committed_frame: base_frame.clone(), - last_sample_frame: base_frame, - last_sample_fingerprint: Some(fingerprint), + worker_pairwise_previous_frame: base_frame.clone(), + last_sample_frame: base_frame.clone(), + last_sample_fingerprint: Some(fingerprint.clone()), + last_downward_observed_frame: base_frame, + last_downward_observed_fingerprint: Some(fingerprint), + last_preview_only_downward_local_sample: None, + seeded_preview_only_local_after_observed_burst_commit: false, + pending_unresolved_burst_registered_growth_viewport_top_y: None, + last_blocked_preview_only_local_candidate: None, + pending_suppressed_huge_preview_only_local_followup: None, + pending_suppressed_huge_preview_only_local_followup_remaining_blocks: 0, + pending_extreme_preview_only_local_tail_followup: None, + pending_extreme_preview_only_local_tail_followup_remaining_blocks: 0, last_unconfirmed_upward_fingerprint: None, last_motion_rows_hint: None, + transient_motion_rows_hint: None, + transient_burst_search_enabled: false, + last_downward_sample_registration_result: None, + last_downward_sample_registration_source: None, + last_downward_sample_registration_motion_rows: None, + last_downward_sample_registration_provisional_viewport_top_y: None, + last_observed_sample_registration_result: None, + last_observed_sample_registration_reason: None, + last_observed_sample_registration_motion_rows: None, + last_observed_sample_registration_mean_abs_diff_x100: None, + last_preview_only_local_registration_result: None, + last_preview_only_local_registration_reason: None, + last_preview_only_local_registration_motion_rows: None, + last_preview_only_local_registration_mean_abs_diff_x100: None, + last_downward_viewport_candidate_count: None, + last_downward_viewport_candidates_before_prune: None, + last_downward_viewport_candidates_after_prune: None, + blocked_underconsumed_observed_recovery_in_burst: false, + blocked_lagging_exactly_corroborated_preview_local_tail_in_burst: false, + blocked_followup_after_suppressed_huge_preview_local_jump: false, + blocked_followup_after_extreme_preview_local_tail: false, + blocked_far_committed_only_recovery_after_corroborated_huge_local_jump: false, + consecutive_transient_burst_missing_downward_candidate_frames: 0, + last_block_reason: None, + last_sample_eval_last_motion_rows_hint: None, + last_sample_eval_transient_motion_rows_hint: None, + last_sample_eval_effective_motion_rows_hint: None, + last_sample_eval_transient_burst_search_enabled: false, current_viewport_top_y: 0, observed_viewport_top_y: 0, resume_frontier_top_y: None, @@ -422,14 +591,185 @@ impl ScrollSession { &mut self, frame: RgbaImage, ) -> Result { - self.observe_sample(frame, ScrollDirection::Down) + self.observe_downward_sample_with_motion_hint(frame, None) + } + + pub(crate) fn observe_downward_sample_with_motion_hint( + &mut self, + frame: RgbaImage, + motion_rows_hint: Option, + ) -> Result { + self.observe_downward_sample_with_motion_hint_and_burst(frame, motion_rows_hint, false) + } + + pub(crate) fn observe_downward_sample_with_motion_hint_and_burst( + &mut self, + frame: RgbaImage, + motion_rows_hint: Option, + allow_burst_search: bool, + ) -> Result { + self.observe_sample_with_motion_context( + frame, + ScrollDirection::Down, + motion_rows_hint, + allow_burst_search, + ) } + #[cfg(test)] pub(crate) fn observe_upward_sample( &mut self, frame: RgbaImage, ) -> Result { - self.observe_sample(frame, ScrollDirection::Up) + self.observe_sample_with_motion_context(frame, ScrollDirection::Up, None, false) + } + + pub(crate) fn observe_worker_pairwise_vision_frame( + &mut self, + frame: RgbaImage, + ) -> Result { + self.clear_last_downward_sample_registration(); + + if frame.width() != self.anchor_frame.width() { + return Err(eyre::eyre!( + "frame width mismatch: expected {} got {}", + self.anchor_frame.width(), + frame.width() + )); + } + + let fingerprint = scroll_capture_fingerprint(&frame); + let previous_worker_frame = self.worker_pairwise_previous_frame.clone(); + + if frame == previous_worker_frame { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + self.worker_pairwise_previous_frame = frame; + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_no_change", + ScrollDirection::Down, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some("frame_matches_last_committed_frame"), + ); + + return Ok(ScrollObserveOutcome::NoChange); + } + + let Some(matched) = + classify_vision_downward_sample_motion_against(&previous_worker_frame, &frame) + else { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + self.worker_pairwise_previous_frame = frame; + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_no_change", + ScrollDirection::Down, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some("worker_pairwise_vision_no_downward_offset"), + ); + + return Ok(ScrollObserveOutcome::NoChange); + }; + let corroborated_shift_rows = + estimate_pairwise_downward_shift_rows(&previous_worker_frame, &frame); + if matched.motion_rows >= WORKER_PAIRWISE_CORROBORATION_MIN_ROWS + && corroborated_shift_rows.is_none_or(|estimated| { + estimated == 0 + || matched.motion_rows.abs_diff(estimated) + > WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS + }) { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + self.worker_pairwise_previous_frame = frame; + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: matched.motion_rows, + }), + Some(self.current_viewport_top_y), + Some(0), + Some("worker_pairwise_large_growth_missing_corroboration"), + ); + + return Ok(ScrollObserveOutcome::NoChange); + } + + let candidate_viewport_top_y = self + .current_viewport_top_y + .saturating_add(i32::try_from(matched.motion_rows).unwrap_or_default()); + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + let frame_max_growth_rows = frame.height().saturating_sub(1).max(1); + + if growth_rows == 0 || growth_rows > frame_max_growth_rows { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + self.worker_pairwise_previous_frame = frame; + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: matched.motion_rows, + }), + Some(candidate_viewport_top_y), + Some(growth_rows), + Some("worker_pairwise_growth_exceeded_frame_bounds"), + ); + + return Ok(ScrollObserveOutcome::NoChange); + } + + self.log_decision( + "scroll_capture.worker_pairwise_growth_candidate", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: matched.motion_rows, + }), + Some(candidate_viewport_top_y), + Some(growth_rows), + Some("worker_pairwise_vision"), + ); + self.worker_pairwise_previous_frame = frame.clone(); + self.clear_preview_only_downward_recovery_carryover(); + + self.apply_growth( + frame.clone(), + growth_rows, + candidate_viewport_top_y, + "worker_pairwise_vision", + Some(matched.motion_rows), + None, + None, + ) + } + + fn observe_sample_with_motion_context( + &mut self, + frame: RgbaImage, + input_direction: ScrollDirection, + motion_rows_hint: Option, + allow_burst_search: bool, + ) -> Result { + let previous_hint = self.transient_motion_rows_hint; + let previous_burst = self.transient_burst_search_enabled; + self.transient_motion_rows_hint = motion_rows_hint; + self.transient_burst_search_enabled = allow_burst_search; + self.record_last_sample_eval_context(); + let result = self.observe_sample(frame, input_direction); + self.transient_motion_rows_hint = previous_hint; + self.transient_burst_search_enabled = previous_burst; + result } fn observe_sample( @@ -437,6 +777,8 @@ impl ScrollSession { frame: RgbaImage, input_direction: ScrollDirection, ) -> Result { + self.clear_last_downward_sample_registration(); + if frame.width() != self.anchor_frame.width() { return Err(eyre::eyre!( "frame width mismatch: expected {} got {}", @@ -444,6 +786,20 @@ impl ScrollSession { frame.width() )); } + let use_resume_local_sample = matches!(input_direction, ScrollDirection::Down) + && self.resume_frontier_top_y.is_some(); + + if self.matches_downward_no_change_frame(input_direction, use_resume_local_sample, &frame) { + self.log_decision( + "scroll_capture.sample_no_change", + input_direction, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some("frame_matches_last_downward_observed_frame"), + ); + return Ok(ScrollObserveOutcome::NoChange); + } let fingerprint = scroll_capture_fingerprint(&frame); @@ -451,18 +807,62 @@ impl ScrollSession { && self.resume_frontier_top_y.is_some() && self.last_unconfirmed_upward_fingerprint.as_deref() == Some(fingerprint.as_slice()) { + self.log_decision( + "scroll_capture.sample_no_change", + input_direction, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some("frame_matches_last_unconfirmed_upward_fingerprint"), + ); return Ok(ScrollObserveOutcome::NoChange); } - let sample_delta = self - .last_sample_fingerprint - .as_ref() - .map(|previous| scroll_capture_fingerprint_delta(previous, &fingerprint)); - let sample_motion = self.classify_sample_motion(&frame); + let sample_delta = + self.sample_delta_for_input(input_direction, use_resume_local_sample, &fingerprint); + let (sample_motion, downward_sample_match) = + self.classify_input_sample_motion(input_direction, use_resume_local_sample, &frame); let preview_changed = sample_delta.is_some_and(|delta| delta > 0) || sample_motion.is_some(); + tracing::info!( + op = "scroll_capture.sample_eval", + input_direction = ?input_direction, + use_resume_local_sample, + sample_delta, + sample_motion_direction = ?sample_motion.map(|motion| motion.direction), + sample_motion_rows = ?sample_motion.map(|motion| motion.motion_rows), + preview_changed, + frame_equals_last_sample = frame == self.last_sample_frame, + frame_equals_last_downward_observed = frame == self.last_downward_observed_frame, + frame_equals_last_preview_only_downward_local = self + .last_preview_only_downward_local_sample + .as_ref() + .is_some_and(|previous| frame == previous.frame), + last_preview_only_downward_local_viewport_top_y = ?self + .last_preview_only_downward_local_sample + .as_ref() + .map(|sample| sample.viewport_top_y), + frame_equals_last_committed = frame == self.last_committed_frame, + last_motion_rows_hint = ?self.last_motion_rows_hint, + transient_motion_rows_hint = ?self.transient_motion_rows_hint, + effective_motion_rows_hint = ?self.effective_motion_rows_hint(), + current_viewport_top_y = self.current_viewport_top_y, + observed_viewport_top_y = self.observed_viewport_top_y, + resume_frontier_top_y = ?self.resume_frontier_top_y, + resume_frontier_requires_reacquire = self.resume_frontier_requires_reacquire, + "Scroll-capture session evaluated a sampled frame before commit resolution." + ); + if !preview_changed { + self.log_decision( + "scroll_capture.sample_no_change", + input_direction, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some("sample_delta_and_motion_both_absent"), + ); return Ok(ScrollObserveOutcome::NoChange); } if matches!(input_direction, ScrollDirection::Up) { @@ -475,12 +875,115 @@ impl ScrollSession { ); } - let previous_sample_frame = self.last_sample_frame.clone(); - let previous_sample_fingerprint = self.last_sample_fingerprint.clone(); + self.observe_downward_input(frame, sample_motion, downward_sample_match, preview_changed) + } - self.last_unconfirmed_upward_fingerprint = None; + fn matches_downward_no_change_frame( + &self, + input_direction: ScrollDirection, + use_resume_local_sample: bool, + frame: &RgbaImage, + ) -> bool { + matches!(input_direction, ScrollDirection::Down) + && !use_resume_local_sample + && frame + == if self.initial_downward_bootstrap_active() { + &self.last_sample_frame + } else { + &self.last_downward_observed_frame + } + } - self.record_last_sample(&frame, fingerprint); + fn sample_delta_for_input( + &self, + input_direction: ScrollDirection, + use_resume_local_sample: bool, + fingerprint: &[u8], + ) -> Option { + match (input_direction, use_resume_local_sample) { + (ScrollDirection::Down, true) => self + .last_sample_fingerprint + .as_ref() + .map(|previous| scroll_capture_fingerprint_delta(previous, fingerprint)), + (ScrollDirection::Down, false) => self + .last_downward_observed_fingerprint + .as_ref() + .map(|previous| scroll_capture_fingerprint_delta(previous, fingerprint)), + (ScrollDirection::Up, _) => self + .last_sample_fingerprint + .as_ref() + .map(|previous| scroll_capture_fingerprint_delta(previous, fingerprint)), + } + } + + fn classify_input_sample_motion( + &mut self, + input_direction: ScrollDirection, + use_resume_local_sample: bool, + frame: &RgbaImage, + ) -> (Option, Option) { + if matches!(input_direction, ScrollDirection::Up) { + return (self.classify_sample_motion(frame), None); + } + + let downward_registration = if use_resume_local_sample { + self.classify_downward_sample_motion_against(&self.last_sample_frame, frame) + .0 + .map_source(DownwardSampleMatchSource::ObservedSample) + } else { + self.classify_downward_sample_motion_with_local_recovery(frame) + }; + + match downward_registration { + DownwardRegistrationWithSource::Matched(matched) => { + self.record_last_downward_sample_registration( + "matched", + Some(matched.source), + Some(matched.matched.motion_rows), + ); + ( + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: matched.matched.motion_rows, + }), + Some(matched), + ) + }, + DownwardRegistrationWithSource::Ambiguous { best, .. } => { + self.record_last_downward_sample_registration( + "ambiguous", + Some(best.source), + Some(best.matched.motion_rows), + ); + self.log_decision( + "scroll_capture.down_input_ambiguous_registration", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: best.matched.motion_rows, + }), + Some(self.current_viewport_top_y), + Some(0), + Some("sample_downward_registration_competed_with_far_apart_candidate"), + ); + + (None, None) + }, + DownwardRegistrationWithSource::NoMatch => { + self.record_last_downward_sample_registration("no_match", None, None); + (None, None) + }, + } + } + + fn observe_downward_input( + &mut self, + frame: RgbaImage, + sample_motion: Option, + downward_sample_match: Option, + preview_changed: bool, + ) -> Result { + self.last_unconfirmed_upward_fingerprint = None; if let Some(motion) = sample_motion { match motion.direction { @@ -489,13 +992,13 @@ impl ScrollSession { &self.last_committed_frame, &frame, ScrollDirection::Down, - self.last_motion_rows_hint, + self.effective_motion_rows_hint(), ); let committed_up_match = self.evaluate_reference_overlap_direction( &self.last_committed_frame, &frame, ScrollDirection::Up, - self.last_motion_rows_hint, + self.effective_motion_rows_hint(), ); if let Some(up_match) = upward_confirmation_match_for_downward_input( @@ -503,53 +1006,45 @@ impl ScrollSession { committed_down_match, self.current_viewport_top_y > 0, ) { - self.observe_upward_rewind_from_committed(up_match.motion_rows); - self.log_decision( + return Ok(self.fail_closed_downward_non_monotonic_frame( + preview_changed, + self.last_sample_frame.clone(), + self.last_sample_fingerprint.clone(), "scroll_capture.down_input_detected_upward_motion", - input_direction, - Some(MotionObservation { + MotionObservation { direction: ScrollDirection::Up, motion_rows: up_match.motion_rows, - }), - None, - None, - Some( - "downward_input_confirmed_upward_motion_with_last_committed_match", - ), - ); - - return Ok(ScrollObserveOutcome::UnsupportedDirection { - direction: ScrollDirection::Up, - }); + }, + "downward_input_confirmed_upward_motion_with_last_committed_match", + )); } - self.log_decision( + return Ok(self.fail_closed_downward_non_monotonic_frame( + preview_changed, + self.last_sample_frame.clone(), + self.last_sample_fingerprint.clone(), "scroll_capture.down_input_detected_upward_motion", - input_direction, - Some(motion), - None, - None, - Some("downward_input_upward_motion_lacked_committed_support"), - ); - - return Ok(preview_update_outcome(preview_changed)); + motion, + "downward_input_upward_motion_lacked_committed_support", + )); }, ScrollDirection::Down => { return self.observe_downward_motion( frame, - motion.motion_rows, + downward_sample_match.unwrap_or(DownwardSampleMatch { + matched: DirectionMatch { + mean_abs_diff_x100: u32::MAX, + motion_rows: motion.motion_rows, + }, + source: DownwardSampleMatchSource::ObservedSample, + }), preview_changed, ); }, } } - self.observe_fallback_downward_growth( - frame, - preview_changed, - previous_sample_frame, - previous_sample_fingerprint, - ) + self.observe_fallback_downward_growth(frame, preview_changed) } fn observe_upward_input( @@ -751,27 +1246,28 @@ impl ScrollSession { } fn diagnose_upward_input(&self, frame: &RgbaImage) -> UpwardInputDiagnostics { + let effective_motion_rows_hint = self.effective_motion_rows_hint(); let sample_down_match_eval = self.diagnose_reference_overlap_direction( &self.last_sample_frame, frame, ScrollDirection::Down, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); let sample_up_match_eval = self.diagnose_upward_reference_overlap_direction( &self.last_sample_frame, frame, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); let committed_down_match_eval = self.diagnose_reference_overlap_direction( &self.last_committed_frame, frame, ScrollDirection::Down, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); let committed_up_match_eval = self.diagnose_upward_reference_overlap_direction( &self.last_committed_frame, frame, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); UpwardInputDiagnostics { @@ -915,7 +1411,7 @@ impl ScrollSession { } fn log_decision( - &self, + &mut self, op: &'static str, input_direction: ScrollDirection, detected_motion: Option, @@ -923,6 +1419,7 @@ impl ScrollSession { growth_rows: Option, block_reason: Option<&'static str>, ) { + self.last_block_reason = block_reason; tracing::info!( op, input_direction = ?input_direction, @@ -972,6 +1469,8 @@ impl ScrollSession { tracing::info!( op = "scroll_capture.up_input_search_window_eval", last_motion_rows_hint = ?self.last_motion_rows_hint, + transient_motion_rows_hint = ?self.transient_motion_rows_hint, + effective_motion_rows_hint = ?self.effective_motion_rows_hint(), sample_delta = ?log.sample_delta, frame_equals_last_sample = log.frame_equals_last_sample, frame_equals_last_committed = log.frame_equals_last_committed, @@ -1059,11 +1558,63 @@ impl ScrollSession { self.last_sample_fingerprint = Some(fingerprint); } + fn record_last_downward_observed_sample(&mut self, frame: &RgbaImage, fingerprint: Vec) { + self.last_downward_observed_frame = frame.clone(); + self.last_downward_observed_fingerprint = Some(fingerprint); + } + + fn record_preview_only_downward_local_sample( + &mut self, + frame: &RgbaImage, + viewport_top_y: i32, + ) { + self.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: frame.clone(), viewport_top_y }); + } + + fn clear_preview_only_downward_local_sample(&mut self) { + self.last_preview_only_downward_local_sample = None; + self.seeded_preview_only_local_after_observed_burst_commit = false; + self.pending_unresolved_burst_registered_growth_viewport_top_y = None; + self.last_blocked_preview_only_local_candidate = None; + } + + fn clear_preview_only_downward_recovery_carryover(&mut self) { + self.clear_preview_only_downward_local_sample(); + self.pending_suppressed_huge_preview_only_local_followup = None; + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 0; + self.pending_extreme_preview_only_local_tail_followup = None; + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 0; + } + fn restore_last_sample(&mut self, frame: RgbaImage, fingerprint: Option>) { self.last_sample_frame = frame; self.last_sample_fingerprint = fingerprint; } + fn fail_closed_downward_non_monotonic_frame( + &mut self, + preview_changed: bool, + previous_sample_frame: RgbaImage, + previous_sample_fingerprint: Option>, + op: &'static str, + detected_motion: MotionObservation, + block_reason: &'static str, + ) -> ScrollObserveOutcome { + self.restore_last_sample(previous_sample_frame, previous_sample_fingerprint); + self.clear_preview_only_downward_local_sample(); + self.log_decision( + op, + ScrollDirection::Down, + Some(detected_motion), + Some(self.current_viewport_top_y), + Some(0), + Some(block_reason), + ); + + preview_update_outcome(preview_changed) + } + fn observe_upward_rewind(&mut self, motion_rows: u32) { let motion_rows = i32::try_from(motion_rows).unwrap_or(i32::MAX); @@ -1088,6 +1639,7 @@ impl ScrollSession { frontier_top_y: i32, ) { self.last_motion_rows_hint = None; + self.clear_preview_only_downward_local_sample(); self.resume_frontier_requires_reacquire = true; self.resume_frontier_top_y.get_or_insert(frontier_top_y); @@ -1097,6 +1649,7 @@ impl ScrollSession { fn observe_unconfirmed_upward_rewind(&mut self) { self.last_motion_rows_hint = None; + self.clear_preview_only_downward_local_sample(); let frontier_top_y = self.current_viewport_top_y; @@ -1110,10 +1663,10 @@ impl ScrollSession { fn observe_downward_motion( &mut self, frame: RgbaImage, - motion_rows: u32, + observed_match: DownwardSampleMatch, preview_changed: bool, ) -> Result { - self.last_motion_rows_hint = Some(motion_rows); + let motion_rows = observed_match.matched.motion_rows; if self.resume_frontier_top_y.is_some() { return self.observe_downward_motion_while_resume_frontier_active( @@ -1123,27 +1676,377 @@ impl ScrollSession { ); } - let candidate_viewport_top_y = self - .observed_viewport_top_y - .saturating_add(i32::try_from(motion_rows).unwrap_or_default()); - - self.observe_downward_growth_to_viewport( - frame, - candidate_viewport_top_y, - preview_changed, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - "sample_motion_downward_growth", - ) - } - - fn observe_downward_motion_while_resume_frontier_active( - &mut self, - frame: RgbaImage, - motion_rows: u32, - preview_changed: bool, - ) -> Result { - let candidate_observed_viewport_top_y = self - .observed_viewport_top_y + let candidate = match self.resolve_downward_viewport_candidate(&frame, observed_match) { + DownwardViewportResolution::NoMatch => { + let reset_preview_only_local_baseline = + self.should_reset_preview_only_local_baseline_after_huge_far_committed_block(); + let preview_only_local_viewport_top_y = if self + .blocked_underconsumed_observed_recovery_in_burst + || self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst + || self.blocked_followup_after_suppressed_huge_preview_local_jump + || self.blocked_followup_after_extreme_preview_local_tail + || self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump + { + self.stable_preview_only_downward_local_viewport_top_y() + } else { + self.preview_only_downward_local_viewport_top_y_for_sample_match(observed_match) + }; + let block_reason = if self.blocked_underconsumed_observed_recovery_in_burst { + "underconsumed_observed_recovery_under_transient_burst" + } else if self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst { + "lagging_exactly_corroborated_preview_local_tail_under_transient_burst" + } else if self.blocked_followup_after_suppressed_huge_preview_local_jump { + "followup_after_suppressed_huge_preview_local_jump_under_transient_burst" + } else if self.blocked_followup_after_extreme_preview_local_tail { + "followup_after_extreme_preview_local_tail_under_transient_burst" + } else if self + .blocked_far_committed_only_recovery_after_corroborated_huge_local_jump + { + "far_committed_only_recovery_after_corroborated_huge_local_jump_under_transient_burst" + } else { + "no_downward_viewport_candidate_resolved" + }; + self.pending_unresolved_burst_registered_growth_viewport_top_y = if block_reason + == "no_downward_viewport_candidate_resolved" + && self.last_downward_sample_registration_result == Some("matched") + { + self.last_downward_sample_registration_provisional_viewport_top_y.filter( + |viewport_top_y| { + self.transient_burst_growth_matches_pending_hint_band(*viewport_top_y) + }, + ) + } else { + None + }; + self.consecutive_transient_burst_missing_downward_candidate_frames = if self + .transient_burst_search_enabled + && preview_only_local_viewport_top_y.is_some() + { + self.consecutive_transient_burst_missing_downward_candidate_frames + .saturating_add(1) + } else { + 0 + }; + self.refresh_local_downward_sample(&frame); + if self.should_refresh_downward_observed_baseline_after_huge_suppressed_jump() { + self.record_last_downward_observed_sample( + &frame, + scroll_capture_fingerprint(&frame), + ); + } + if reset_preview_only_local_baseline { + self.clear_preview_only_downward_local_sample(); + } else { + self.refresh_preview_only_downward_local_sample( + &frame, + preview_only_local_viewport_top_y, + ); + } + self.log_decision( + "scroll_capture.downward_viewport_authority_missing", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + None, + Some(0), + Some(block_reason), + ); + return Ok(preview_update_outcome(preview_changed)); + }, + DownwardViewportResolution::Selected(candidate) => candidate, + DownwardViewportResolution::Ambiguous { preferred, competing } => { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.preview_only_downward_local_viewport_top_y_for_sample_match( + observed_match, + ), + ); + self.log_decision( + "scroll_capture.downward_viewport_authority_ambiguous", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: preferred.motion_rows, + }), + Some(preferred.viewport_top_y), + Some(0), + Some(preferred.competing_block_reason(competing)), + ); + + return Ok(preview_update_outcome(preview_changed)); + }, + }; + + if self.should_fail_closed_tiny_observed_recovery_in_burst(candidate) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("tiny_observed_recovery_under_transient_burst"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + if self.should_fail_closed_outsized_observed_recovery_after_one_pixel_preview_local_commit( + candidate, + ) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("outsized_observed_recovery_after_one_pixel_preview_local_commit"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + if self.should_fail_closed_tiny_preview_only_local_recovery_in_burst(candidate) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("tiny_preview_only_local_recovery_under_transient_burst"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + if self + .should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst(candidate) + { + self.pending_extreme_preview_only_local_tail_followup = Some(candidate); + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 1; + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("exactly_corroborated_preview_local_tail_under_extreme_transient_burst"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + if self.should_fail_closed_preview_only_local_tail_after_unresolved_burst(candidate) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("preview_only_local_tail_after_unresolved_transient_burst"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + if self.should_fail_closed_tiny_committed_keyframe_recovery_in_burst(candidate) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some("tiny_committed_keyframe_recovery_under_transient_burst"), + ); + return Ok(preview_update_outcome(preview_changed)); + } + + self.observe_downward_growth_to_viewport( + frame, + candidate.viewport_top_y, + preview_changed, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: candidate.motion_rows, + }), + candidate.source.decision_source(), + ) + } + + fn should_fail_closed_tiny_observed_recovery_in_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.source == DownwardViewportCandidateSource::ObservedSample + && candidate.motion_rows <= TINY_OBSERVED_BURST_RECOVERY_MAX_MOTION_ROWS + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self + .last_motion_rows_hint + .is_some_and(|last_hint| last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS) + && self.last_preview_only_downward_local_sample.is_none() + } + + fn should_fail_closed_outsized_observed_recovery_after_one_pixel_preview_local_commit( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.source == DownwardViewportCandidateSource::ObservedSample + && candidate.motion_rows >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS.saturating_mul(2) + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self.last_motion_rows_hint == Some(1) + && self.growth_history.last().is_some_and(|commit| { + commit.growth_rows == 1 + && commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + }) + } + + fn should_fail_closed_tiny_preview_only_local_recovery_in_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if self.seeded_preview_only_local_catch_up_candidate_can_commit(candidate) { + return false; + } + let small_recovery_lags_recent_continuity = + self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && candidate.motion_rows + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_div(2) + && candidate.motion_rows + < last_hint.saturating_sub(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + }); + + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && candidate.motion_rows <= TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS + && candidate.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(2) + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MIN_LAST_HINT_ROWS + }) || candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && small_recovery_lags_recent_continuity + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + } + + fn should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(transient_motion_rows_hint) = self.normalized_transient_motion_rows_hint() else { + return false; + }; + + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && candidate.motion_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && candidate.motion_rows >= last_motion_rows_hint.saturating_mul(2) + && transient_motion_rows_hint + >= last_motion_rows_hint + .saturating_mul(EXTREME_TRANSIENT_PREVIEW_LOCAL_TAIL_MULTIPLIER) + && self.last_observed_sample_registration_result == Some("matched") + && self.last_observed_sample_registration_motion_rows == Some(candidate.motion_rows) + && self.growth_history.iter().rev().take(2).count() == 2 + && self.growth_history.iter().rev().take(2).all(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + }) && self.last_downward_viewport_candidates_before_prune.as_ref().is_some_and(|value| { + let exact_committed = format!( + "CommittedKeyframe@{}/{}:", + candidate.viewport_top_y, candidate.motion_rows + ); + value.contains(&exact_committed) + }) + } + + fn should_fail_closed_preview_only_local_tail_after_unresolved_burst( + &mut self, + candidate: DownwardViewportCandidate, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(transient_motion_rows_hint) = self.normalized_transient_motion_rows_hint() else { + return false; + }; + let candidate_is_extreme_preview_local_tail = candidate.source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && self.last_block_reason == Some("no_downward_viewport_candidate_resolved") + && self.transient_burst_search_enabled; + let unresolved_burst_has_registered_growth_in_pending_band = + candidate_is_extreme_preview_local_tail + && self + .pending_unresolved_burst_registered_growth_viewport_top_y + .take() + .is_some_and(|viewport_top_y| { + self.transient_burst_growth_matches_pending_hint_band(viewport_top_y) + }); + + candidate_is_extreme_preview_local_tail + && !unresolved_burst_has_registered_growth_in_pending_band + && candidate.motion_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && candidate.motion_rows >= last_motion_rows_hint.saturating_mul(2) + && transient_motion_rows_hint + >= last_motion_rows_hint + .saturating_mul(EXTREME_TRANSIENT_PREVIEW_LOCAL_TAIL_MULTIPLIER) + && self + .last_preview_only_downward_local_sample + .as_ref() + .is_some_and(|sample| sample.viewport_top_y == self.current_viewport_top_y) + } + + fn should_fail_closed_tiny_committed_keyframe_recovery_in_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && growth_rows <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && candidate.motion_rows + > growth_rows.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS.saturating_mul(2)) + && candidate.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(2) + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS.saturating_add(2) + }) + } + + fn observe_downward_motion_while_resume_frontier_active( + &mut self, + frame: RgbaImage, + motion_rows: u32, + preview_changed: bool, + ) -> Result { + let candidate_observed_viewport_top_y = self + .observed_viewport_top_y .saturating_add(i32::try_from(motion_rows).unwrap_or_default()); let Some(resume_frontier_top_y) = self.resume_frontier_top_y else { return Ok(preview_update_outcome(preview_changed)); @@ -1556,11 +2459,26 @@ impl ScrollSession { detected_motion: Option, decision_source: &'static str, ) -> Result { - self.observed_viewport_top_y = candidate_viewport_top_y; - let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + let effective_motion_rows_hint = self.effective_motion_rows_hint(); + self.pending_unresolved_burst_registered_growth_viewport_top_y = None; + + if self.bootstrap_initial_growth_cap_rows().is_some_and(|cap| growth_rows > cap) { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + detected_motion, + Some(candidate_viewport_top_y), + Some(growth_rows), + Some("bootstrap_growth_exceeded_initial_growth_cap"), + ); + + return Ok(preview_update_outcome(preview_changed)); + } if growth_rows == 0 { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; let block_reason = if self.resume_frontier_top_y.is_some() { Some("candidate_viewport_did_not_pass_resume_frontier") } else { @@ -1578,7 +2496,21 @@ impl ScrollSession { return Ok(preview_update_outcome(preview_changed)); } + let max_growth_rows = self.max_downward_growth_rows_for_frame(&frame); + + if growth_rows > max_growth_rows { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + detected_motion, + Some(candidate_viewport_top_y), + Some(growth_rows), + Some("candidate_viewport_growth_exceeded_monotonic_cap"), + ); + return Ok(preview_update_outcome(preview_changed)); + } self.log_decision( "scroll_capture.downward_growth_candidate", ScrollDirection::Down, @@ -1587,22 +2519,55 @@ impl ScrollSession { Some(growth_rows), Some(decision_source), ); + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + let previous_motion_rows_hint = self.last_motion_rows_hint; + self.last_motion_rows_hint = Some(growth_rows); + + self.apply_growth( + frame, + growth_rows, + candidate_viewport_top_y, + decision_source, + detected_motion.map(|motion| motion.motion_rows), + effective_motion_rows_hint, + previous_motion_rows_hint, + ) + } + + fn max_downward_growth_rows_for_frame(&self, frame: &RgbaImage) -> u32 { + let config = OverlapSearchConfig::default(); + let effective_min_overlap = if frame.height() <= config.min_overlap_rows { + 1 + } else { + config.min_overlap_rows.max(1) + }; + let frame_max_growth_rows = frame.height().saturating_sub(effective_min_overlap).max(1); + + if self.transient_burst_search_enabled { + return self + .transient_motion_rows_hint + .map(|hint| { + frame_max_growth_rows.min(hint.max(INITIAL_DOWNWARD_MAX_MOTION_ROWS)).max(1) + }) + .unwrap_or(frame_max_growth_rows.clamp(1, INITIAL_DOWNWARD_MAX_MOTION_ROWS)); + } - self.apply_growth(frame, growth_rows, candidate_viewport_top_y) + frame_max_growth_rows.clamp(1, INITIAL_DOWNWARD_MAX_MOTION_ROWS) } fn classify_sample_motion(&self, frame: &RgbaImage) -> Option { + let effective_motion_rows_hint = self.effective_motion_rows_hint(); let down_match = self.evaluate_reference_overlap_direction( &self.last_sample_frame, frame, ScrollDirection::Down, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); let up_match = self.evaluate_reference_overlap_direction( &self.last_sample_frame, frame, ScrollDirection::Up, - self.last_motion_rows_hint, + effective_motion_rows_hint, ); match (down_match, up_match) { @@ -1637,63 +2602,682 @@ impl ScrollSession { } } - pub(crate) fn preview_image(&self) -> &RgbaImage { - &self.preview_image - } + fn classify_downward_sample_motion( + &self, + frame: &RgbaImage, + ) -> (DownwardRegistration, Option<&'static str>) { + let previous = if self.initial_downward_bootstrap_active() { + &self.last_sample_frame + } else { + &self.last_downward_observed_frame + }; - pub(crate) fn export_image(&self) -> &RgbaImage { - &self.export_image + self.classify_downward_sample_motion_against(previous, frame) } - pub(crate) fn export_dimensions(&self) -> (u32, u32) { - self.export_image.dimensions() + fn classify_downward_sample_motion_with_local_recovery( + &mut self, + frame: &RgbaImage, + ) -> DownwardRegistrationWithSource { + let (primary_raw, primary_reason) = self.classify_downward_sample_motion(frame); + let primary = primary_raw.map_source(DownwardSampleMatchSource::ObservedSample); + self.record_registration_diagnostics( + DownwardSampleMatchSource::ObservedSample, + primary, + primary_reason, + ); + let Some(previous_local) = self.last_preview_only_downward_local_sample.as_ref() else { + return primary; + }; + + let (local_raw, local_reason) = + self.classify_preview_only_local_recovery_motion_against(&previous_local.frame, frame); + let local = local_raw.map_source(DownwardSampleMatchSource::PreviewOnlyLocalSample); + self.record_registration_diagnostics( + DownwardSampleMatchSource::PreviewOnlyLocalSample, + local, + local_reason, + ); + + match (primary, local) { + ( + DownwardRegistrationWithSource::Matched(primary), + DownwardRegistrationWithSource::Matched(local), + ) => { + if self.should_prefer_preview_only_local_recovery_after_extreme_tail_block( + primary, local, + ) { + DownwardRegistrationWithSource::Matched(local) + } else if self + .should_prefer_observed_sample_over_preview_only_local_recovery(primary, local) + { + DownwardRegistrationWithSource::Matched(primary) + } else if self + .should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) + { + DownwardRegistrationWithSource::Matched(local) + } else if local.matched.mean_abs_diff_x100 <= primary.matched.mean_abs_diff_x100 { + DownwardRegistrationWithSource::Matched(local) + } else { + DownwardRegistrationWithSource::Matched(primary) + } + }, + (DownwardRegistrationWithSource::Matched(primary), _) => { + DownwardRegistrationWithSource::Matched(primary) + }, + (_, DownwardRegistrationWithSource::Matched(local)) => { + DownwardRegistrationWithSource::Matched(local) + }, + (primary, _) => primary, + } } - pub(crate) fn undo_last_append(&mut self) -> bool { - let Some(_commit) = self.growth_history.pop() else { + fn should_prefer_observed_sample_over_preview_only_local_recovery( + &self, + primary: DownwardSampleMatch, + local: DownwardSampleMatch, + ) -> bool { + let small_local_recovery_lags_recent_continuity = + self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && local.matched.motion_rows + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_div(2) + && local.matched.motion_rows + < last_hint.saturating_sub(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + }); + + small_local_recovery_lags_recent_continuity + && self.transient_burst_motion_hint_exceeds_local_authority(local.matched.motion_rows) + && primary.matched.motion_rows + > local + .matched + .motion_rows + .saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + && self.last_motion_rows_hint.is_some_and(|last_hint| { + primary + .matched + .motion_rows + .saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + >= last_hint && primary.matched.motion_rows <= last_hint + }) && self + .transient_pending_growth_cap_rows() + .is_some_and(|cap| primary.matched.motion_rows <= cap) + } + + fn should_prefer_preview_only_local_recovery_after_extreme_tail_block( + &self, + primary: DownwardSampleMatch, + local: DownwardSampleMatch, + ) -> bool { + let Some(pending_candidate) = self.pending_extreme_preview_only_local_tail_followup else { return false; }; - let _ = self.bottom_segments.pop(); - let _ = self.bottom_preview_segments.pop(); - let Ok(export_image) = self.rebuild_export_image() else { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; - let Ok(preview_image) = self.rebuild_preview_image() else { + let Some(transient_motion_rows_hint) = self.normalized_transient_motion_rows_hint() else { return false; }; - self.export_image = export_image; - self.preview_image = preview_image; + primary.source == DownwardSampleMatchSource::ObservedSample + && local.source == DownwardSampleMatchSource::PreviewOnlyLocalSample + && primary.matched.motion_rows == pending_candidate.motion_rows + && local.matched.motion_rows >= last_motion_rows_hint + && local.matched.motion_rows < primary.matched.motion_rows + && local.matched.motion_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && transient_motion_rows_hint + >= last_motion_rows_hint + .saturating_mul(EXTREME_TRANSIENT_PREVIEW_LOCAL_TAIL_MULTIPLIER) + } - if let Some(previous) = self.growth_history.last() { - self.last_motion_rows_hint = Some(previous.growth_rows); - self.current_viewport_top_y = previous.viewport_top_y; - self.observed_viewport_top_y = previous.viewport_top_y; - self.last_committed_frame = previous.frame.clone(); - self.last_sample_frame = previous.frame.clone(); - self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&previous.frame)); - self.last_unconfirmed_upward_fingerprint = None; - self.resume_frontier_top_y = None; - self.resume_frontier_requires_reacquire = false; - } else { - self.last_committed_frame = self.anchor_frame.clone(); - self.last_sample_frame = self.anchor_frame.clone(); - self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&self.anchor_frame)); - self.last_unconfirmed_upward_fingerprint = None; - self.last_motion_rows_hint = None; - self.current_viewport_top_y = 0; - self.observed_viewport_top_y = 0; - self.resume_frontier_top_y = None; - self.resume_frontier_requires_reacquire = false; - } + fn should_prefer_preview_only_local_recovery_over_observed_sample( + &self, + primary: DownwardSampleMatch, + local: DownwardSampleMatch, + ) -> bool { + self.transient_burst_search_enabled + && self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && (local.matched.motion_rows >= last_hint + || (local.matched.motion_rows + <= TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS + && self.consecutive_transient_burst_missing_downward_candidate_frames + >= 2)) && local.matched.motion_rows + <= last_hint.saturating_add(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + && primary.matched.motion_rows + > local + .matched + .motion_rows + .saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + }) || self + .preview_only_local_slowdown_tail_followup_can_prefer_observed_override(primary, local) + } + + fn preview_only_local_slowdown_tail_followup_can_prefer_observed_override( + &self, + primary: DownwardSampleMatch, + local: DownwardSampleMatch, + ) -> bool { + self.transient_burst_search_enabled + && self.last_preview_only_downward_local_sample.is_some() + && local.source == DownwardSampleMatchSource::PreviewOnlyLocalSample + && primary.source == DownwardSampleMatchSource::ObservedSample + && self.last_motion_rows_hint.is_some_and(|last_hint| { + let tiny_followup = last_hint <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && local.matched.motion_rows + <= TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS; + let near_continuity_followup = last_hint + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && local.matched.motion_rows + <= last_hint.saturating_add(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS); + + (tiny_followup || near_continuity_followup) + && primary.matched.motion_rows + > local + .matched + .motion_rows + .saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + }) && self.growth_history.last().is_some_and(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && self + .last_motion_rows_hint + .is_some_and(|last_hint| commit.growth_rows <= last_hint) + }) + } - true + fn record_registration_diagnostics( + &mut self, + source: DownwardSampleMatchSource, + registration: DownwardRegistrationWithSource, + reason: Option<&'static str>, + ) { + let (result, motion_rows, mean_abs_diff_x100) = match registration { + DownwardRegistrationWithSource::NoMatch => ("no_match", None, None), + DownwardRegistrationWithSource::Matched(matched) => ( + "matched", + Some(matched.matched.motion_rows), + Some(matched.matched.mean_abs_diff_x100), + ), + DownwardRegistrationWithSource::Ambiguous { best, .. } => { + ("ambiguous", Some(best.matched.motion_rows), Some(best.matched.mean_abs_diff_x100)) + }, + }; + + match source { + DownwardSampleMatchSource::ObservedSample => { + self.last_observed_sample_registration_result = Some(result); + self.last_observed_sample_registration_reason = reason; + self.last_observed_sample_registration_motion_rows = motion_rows; + self.last_observed_sample_registration_mean_abs_diff_x100 = mean_abs_diff_x100; + }, + DownwardSampleMatchSource::PreviewOnlyLocalSample => { + self.last_preview_only_local_registration_result = Some(result); + self.last_preview_only_local_registration_reason = reason; + self.last_preview_only_local_registration_motion_rows = motion_rows; + self.last_preview_only_local_registration_mean_abs_diff_x100 = mean_abs_diff_x100; + }, + } } - fn evaluate_reference_overlap_direction( + fn classify_downward_sample_motion_against( &self, previous: &RgbaImage, - next: &RgbaImage, + frame: &RgbaImage, + ) -> (DownwardRegistration, Option<&'static str>) { + let config = OverlapSearchConfig::default(); + let preferred_ranges = self.sequential_downward_motion_ranges(previous, frame, config); + + let (registration, reason) = self + .evaluate_reference_downward_registration_with_preferred_ranges( + previous, + frame, + self.last_motion_rows_hint, + &preferred_ranges, + self.transient_burst_search_enabled, + ); + + match registration { + DownwardRegistration::Matched(matched) + if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) => + { + (DownwardRegistration::NoMatch, Some("bootstrap_hint_exceeded")) + }, + other => (other, reason), + } + } + + fn classify_preview_only_local_recovery_motion_against( + &self, + previous: &RgbaImage, + frame: &RgbaImage, + ) -> (DownwardRegistration, Option<&'static str>) { + let config = OverlapSearchConfig::default(); + let preferred_range = + self.preview_only_local_recovery_motion_range(previous, frame, config); + let preferred_ranges = preferred_range.into_iter().collect::>(); + let motion_rows_hint = + self.last_motion_rows_hint.or(self.normalized_transient_motion_rows_hint()); + + let (registration, reason) = self + .evaluate_reference_downward_registration_with_preferred_ranges( + previous, + frame, + motion_rows_hint, + &preferred_ranges, + self.transient_burst_search_enabled, + ); + + match registration { + DownwardRegistration::Matched(matched) + if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) => + { + (DownwardRegistration::NoMatch, Some("bootstrap_hint_exceeded")) + }, + other => (other, reason), + } + } + + fn effective_motion_rows_hint(&self) -> Option { + let transient = self.normalized_transient_motion_rows_hint(); + + match (self.last_motion_rows_hint, transient) { + (Some(last), Some(_transient)) => Some(last), + (Some(last), None) => Some(last), + (None, Some(transient)) => Some(transient), + (None, None) => None, + } + } + + fn normalized_transient_motion_rows_hint(&self) -> Option { + let transient = self.transient_motion_rows_hint?; + if self.transient_burst_search_enabled { + return Some(transient); + } + + match self.last_motion_rows_hint { + Some(last) => { + let cap = last + .saturating_mul(TRANSIENT_MOTION_HINT_MAX_MULTIPLIER) + .max(TRANSIENT_MOTION_HINT_MIN_CAP_ROWS) + .max(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS); + + (transient <= cap).then_some(transient) + }, + None => Some(transient.min(INITIAL_DOWNWARD_MAX_MOTION_ROWS)), + } + } + + fn transient_burst_motion_hint_exceeds_local_authority(&self, local_motion_rows: u32) -> bool { + if !self.transient_burst_search_enabled { + return false; + } + + let Some(transient) = self.transient_motion_rows_hint else { + return false; + }; + let capped_local_motion_rows = + local_motion_rows.min(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS); + let local_authority_rows = self + .last_motion_rows_hint + .unwrap_or(capped_local_motion_rows) + .max(capped_local_motion_rows); + let local_authority_cap = local_authority_rows + .saturating_mul(TRANSIENT_MOTION_HINT_MAX_MULTIPLIER) + .max(TRANSIENT_MOTION_HINT_MIN_CAP_ROWS); + + transient > local_authority_cap + } + + fn preview_only_local_recovery_motion_range( + &self, + previous: &RgbaImage, + next: &RgbaImage, + config: OverlapSearchConfig, + ) -> Option> { + let max_motion_rows = max_directional_motion_rows(previous, next, config); + + if max_motion_rows == 0 { + return None; + } + + if self.initial_downward_bootstrap_active() && self.last_motion_rows_hint.is_none() { + if let Some(hint) = self.normalized_transient_motion_rows_hint() + && hint <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + { + let tolerance = (hint / 2) + .clamp(1, PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS) + .min(max_motion_rows); + let min_motion_rows = hint.saturating_sub(tolerance).max(1); + let max_motion_rows = hint + .saturating_add(tolerance) + .min(max_motion_rows) + .min(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS); + + return Some(min_motion_rows..=max_motion_rows); + } + + return Some( + 1..=PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.min(max_motion_rows).max(1), + ); + } + + if let Some(hint) = + self.last_motion_rows_hint.or(self.normalized_transient_motion_rows_hint()) + { + let tolerance = (hint / 2) + .clamp(1, PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS) + .min(max_motion_rows); + let min_motion_rows = if self.seeded_preview_only_local_after_observed_burst_commit + || self.preview_only_local_tail_followup_can_include_one_pixel_recovery() + { + 1 + } else { + hint.saturating_sub(tolerance).max(1) + }; + let max_motion_rows = hint + .saturating_add(tolerance) + .min(max_motion_rows) + .min(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS); + + return Some(min_motion_rows..=max_motion_rows); + } + + Some(1..=PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.min(max_motion_rows).max(1)) + } + + fn preview_only_local_tail_followup_can_include_one_pixel_recovery(&self) -> bool { + self.transient_burst_search_enabled + && self.last_preview_only_downward_local_sample.is_some() + && self + .last_motion_rows_hint + .is_some_and(|last_hint| last_hint <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + && self.growth_history.last().is_some_and(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && commit.growth_rows <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + }) + } + + fn refresh_local_downward_sample(&mut self, frame: &RgbaImage) { + if !self.initial_downward_bootstrap_active() { + return; + } + + self.last_unconfirmed_upward_fingerprint = None; + let fingerprint = scroll_capture_fingerprint(frame); + self.record_last_sample(frame, fingerprint); + } + + fn refresh_preview_only_downward_local_sample( + &mut self, + frame: &RgbaImage, + provisional_viewport_top_y: Option, + ) { + let Some(provisional_viewport_top_y) = provisional_viewport_top_y else { + self.clear_preview_only_downward_local_sample(); + return; + }; + if !self.should_refresh_preview_only_downward_local_sample(frame) { + return; + } + self.last_unconfirmed_upward_fingerprint = None; + self.record_preview_only_downward_local_sample(frame, provisional_viewport_top_y); + } + + fn should_refresh_downward_observed_baseline_after_huge_suppressed_jump(&self) -> bool { + self.pending_suppressed_huge_preview_only_local_followup.is_some() + || self.blocked_followup_after_suppressed_huge_preview_local_jump + || self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump + } + + fn should_reset_preview_only_local_baseline_after_huge_far_committed_block(&self) -> bool { + self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump + } + + fn provisional_viewport_top_y_for_downward_sample_match( + &self, + observed_match: DownwardSampleMatch, + ) -> Option { + let motion_rows = i32::try_from(observed_match.matched.motion_rows).unwrap_or_default(); + + match observed_match.source { + DownwardSampleMatchSource::ObservedSample => { + Some(self.observed_viewport_top_y.saturating_add(motion_rows)) + }, + DownwardSampleMatchSource::PreviewOnlyLocalSample => self + .last_preview_only_downward_local_sample + .as_ref() + .map(|sample| sample.viewport_top_y.saturating_add(motion_rows)), + } + } + + fn preview_only_downward_local_viewport_top_y_for_sample_match( + &self, + observed_match: DownwardSampleMatch, + ) -> Option { + let provisional_viewport_top_y = + self.provisional_viewport_top_y_for_downward_sample_match(observed_match)?; + let candidate = DownwardViewportCandidate { + source: observed_match.source.into(), + viewport_top_y: provisional_viewport_top_y, + motion_rows: observed_match.matched.motion_rows, + mean_abs_diff_x100: observed_match.matched.mean_abs_diff_x100, + }; + + if self.should_suppress_observed_sample_candidate(candidate) + || self.should_suppress_preview_only_local_candidate(candidate) + { + return self.stable_preview_only_downward_local_viewport_top_y(); + } + + Some(provisional_viewport_top_y) + } + + fn stable_preview_only_downward_local_viewport_top_y(&self) -> Option { + self.last_preview_only_downward_local_sample + .as_ref() + .map(|sample| sample.viewport_top_y) + .or(Some(self.observed_viewport_top_y)) + } + + fn should_refresh_preview_only_downward_local_sample(&self, frame: &RgbaImage) -> bool { + if self.resume_frontier_top_y.is_some() || self.resume_frontier_requires_reacquire { + return false; + } + if self.last_sample_frame != self.last_downward_observed_frame { + return false; + } + if frame == &self.anchor_frame || frame == &self.last_committed_frame { + return false; + } + if self + .last_preview_only_downward_local_sample + .as_ref() + .is_some_and(|previous| *frame == previous.frame) + { + return false; + } + + !self.growth_history.iter().any(|commit| frame == &commit.frame) + } + + fn initial_downward_bootstrap_active(&self) -> bool { + self.growth_history.is_empty() + && self.current_viewport_top_y == 0 + && self.resume_frontier_top_y.is_none() + && !self.resume_frontier_requires_reacquire + } + + fn bootstrap_motion_cap_from_pending_hint(&self) -> Option { + if !self.initial_downward_bootstrap_active() || self.last_motion_rows_hint.is_some() { + return None; + } + + self.normalized_transient_motion_rows_hint().map(|hint| { + let tolerance = (hint / 2).clamp(1, PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS); + + hint.saturating_add(tolerance) + }) + } + + fn bootstrap_motion_exceeds_pending_hint(&self, motion_rows: u32) -> bool { + self.bootstrap_motion_cap_from_pending_hint().is_some_and(|cap| motion_rows > cap) + } + + fn bootstrap_initial_growth_cap_rows(&self) -> Option { + if !self.initial_downward_bootstrap_active() || self.last_motion_rows_hint.is_some() { + return None; + } + + self.bootstrap_motion_cap_from_pending_hint() + .map(|cap| cap.min(BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS)) + } + + pub(crate) fn preview_image(&self) -> &RgbaImage { + &self.preview_image + } + + pub(crate) fn preview_display_image(&self) -> RgbaImage { + self.export_image.clone() + } + + pub(crate) fn preview_display_mode(&self) -> &'static str { + "committed" + } + + pub(crate) fn export_image(&self) -> &RgbaImage { + &self.export_image + } + + pub(crate) fn current_viewport_top_y(&self) -> i32 { + self.current_viewport_top_y + } + + pub(crate) fn export_dimensions(&self) -> (u32, u32) { + self.export_image.dimensions() + } + + pub(crate) fn commit_telemetry(&self) -> ScrollCommitTelemetry { + let last_commit = self.growth_history.last(); + + ScrollCommitTelemetry { + current_viewport_top_y: self.current_viewport_top_y, + preview_dimensions: self.preview_image.dimensions(), + export_dimensions: self.export_image.dimensions(), + growth_commit_count: self.growth_history.len(), + preview_segment_count: self.bottom_preview_segments.len(), + export_segment_count: self.bottom_segments.len(), + preview_export_segments_aligned: self.bottom_segments.len() + == self.bottom_preview_segments.len() + && self.bottom_segments.len() == self.growth_history.len(), + last_commit_decision_source: last_commit.map(|commit| commit.decision_source), + last_commit_detected_motion_rows: last_commit + .and_then(|commit| commit.detected_motion_rows), + last_commit_effective_motion_rows_hint: last_commit + .and_then(|commit| commit.effective_motion_rows_hint), + last_block_reason: self.last_block_reason, + last_downward_sample_registration_result: self.last_downward_sample_registration_result, + last_downward_sample_registration_source: self.last_downward_sample_registration_source, + last_downward_sample_registration_motion_rows: self + .last_downward_sample_registration_motion_rows, + last_downward_sample_registration_provisional_viewport_top_y: self + .last_downward_sample_registration_provisional_viewport_top_y, + observed_sample_registration_result: self.last_observed_sample_registration_result, + observed_sample_registration_reason: self.last_observed_sample_registration_reason, + observed_sample_registration_motion_rows: self + .last_observed_sample_registration_motion_rows, + observed_sample_registration_mean_abs_diff_x100: self + .last_observed_sample_registration_mean_abs_diff_x100, + preview_only_local_registration_result: self + .last_preview_only_local_registration_result, + preview_only_local_registration_reason: self + .last_preview_only_local_registration_reason, + preview_only_local_registration_motion_rows: self + .last_preview_only_local_registration_motion_rows, + preview_only_local_registration_mean_abs_diff_x100: self + .last_preview_only_local_registration_mean_abs_diff_x100, + last_downward_viewport_candidate_count: self.last_downward_viewport_candidate_count, + last_downward_viewport_candidates_before_prune: self + .last_downward_viewport_candidates_before_prune + .clone(), + last_downward_viewport_candidates_after_prune: self + .last_downward_viewport_candidates_after_prune + .clone(), + sample_eval_last_motion_rows_hint: self.last_sample_eval_last_motion_rows_hint, + sample_eval_transient_motion_rows_hint: self + .last_sample_eval_transient_motion_rows_hint, + sample_eval_effective_motion_rows_hint: self + .last_sample_eval_effective_motion_rows_hint, + sample_eval_transient_burst_search_enabled: self + .last_sample_eval_transient_burst_search_enabled, + preview_only_local_viewport_top_y: self + .last_preview_only_downward_local_sample + .as_ref() + .map(|sample| sample.viewport_top_y), + last_preview_segment_height_px: self + .bottom_preview_segments + .last() + .map(RgbaImage::height), + last_export_segment_height_px: self.bottom_segments.last().map(RgbaImage::height), + } + } + + pub(crate) fn undo_last_append(&mut self) -> bool { + let Some(_commit) = self.growth_history.pop() else { + return false; + }; + let _ = self.bottom_segments.pop(); + let _ = self.bottom_preview_segments.pop(); + let Ok(export_image) = self.rebuild_export_image() else { + return false; + }; + let Ok(preview_image) = self.rebuild_preview_image() else { + return false; + }; + + self.export_image = export_image; + self.preview_image = preview_image; + + if let Some(previous) = self.growth_history.last() { + self.last_motion_rows_hint = Some(previous.growth_rows); + self.current_viewport_top_y = previous.viewport_top_y; + self.observed_viewport_top_y = previous.viewport_top_y; + self.last_committed_frame = previous.frame.clone(); + self.worker_pairwise_previous_frame = previous.frame.clone(); + self.last_sample_frame = previous.frame.clone(); + self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&previous.frame)); + self.last_downward_observed_frame = previous.frame.clone(); + self.last_downward_observed_fingerprint = + Some(scroll_capture_fingerprint(&previous.frame)); + self.clear_preview_only_downward_local_sample(); + self.last_unconfirmed_upward_fingerprint = None; + self.resume_frontier_top_y = None; + self.resume_frontier_requires_reacquire = false; + } else { + self.last_committed_frame = self.anchor_frame.clone(); + self.worker_pairwise_previous_frame = self.anchor_frame.clone(); + self.last_sample_frame = self.anchor_frame.clone(); + self.last_sample_fingerprint = Some(scroll_capture_fingerprint(&self.anchor_frame)); + self.last_downward_observed_frame = self.anchor_frame.clone(); + self.last_downward_observed_fingerprint = + Some(scroll_capture_fingerprint(&self.anchor_frame)); + self.clear_preview_only_downward_local_sample(); + self.last_unconfirmed_upward_fingerprint = None; + self.last_motion_rows_hint = None; + self.current_viewport_top_y = 0; + self.observed_viewport_top_y = 0; + self.resume_frontier_top_y = None; + self.resume_frontier_requires_reacquire = false; + } + + true + } + + fn evaluate_reference_overlap_direction( + &self, + previous: &RgbaImage, + next: &RgbaImage, direction: ScrollDirection, motion_rows_hint: Option, ) -> Option { @@ -1701,107 +3285,1571 @@ impl ScrollSession { let preferred_range = self.preferred_motion_range_from_hint(previous, next, motion_rows_hint, config)?; - evaluate_overlap_direction(previous, next, direction, preferred_range, config) + evaluate_overlap_direction(previous, next, direction, preferred_range, config) + } + + fn evaluate_reference_downward_registration( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + allow_full_range_fallback: bool, + ) -> DownwardRegistration { + let config = OverlapSearchConfig::default(); + let preferred_range = self.preferred_downward_motion_range_from_hint( + previous, + next, + motion_rows_hint, + config, + ); + + self.evaluate_reference_downward_registration_with_preferred_range( + previous, + next, + motion_rows_hint, + preferred_range, + allow_full_range_fallback, + ) + } + + fn evaluate_reference_downward_registration_with_preferred_ranges( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + preferred_ranges: &[RangeInclusive], + allow_full_range_fallback: bool, + ) -> (DownwardRegistration, Option<&'static str>) { + let config = OverlapSearchConfig::default(); + let max_overlap = previous.height().min(next.height()); + let max_motion_rows = max_directional_motion_rows(previous, next, config); + let mut candidates = collect_overlap_direction_matches_in_ranges( + previous, + next, + ScrollDirection::Down, + preferred_ranges, + config, + ); + let mut no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; + + if candidates.is_empty() + && allow_full_range_fallback + && (motion_rows_hint.is_none() || self.transient_burst_search_enabled) + { + candidates = collect_overlap_direction_matches( + previous, + next, + ScrollDirection::Down, + 1..=max_motion_rows, + config, + ); + no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; + } + candidates.retain(|matched| { + downward_registration_has_meaningful_overlap(*matched, max_overlap, config) + }); + if candidates.is_empty() { + no_match_reason.get_or_insert("insufficient_overlap"); + } + + let classification = classify_downward_registration_candidates(&candidates); + let upward_veto = self.evaluate_reference_overlap_direction( + previous, + next, + ScrollDirection::Up, + motion_rows_hint, + ); + + match (classification, upward_veto) { + (DownwardRegistration::Matched(down), Some(up)) + if up.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + <= down.mean_abs_diff_x100 => + { + (DownwardRegistration::NoMatch, Some("upward_veto")) + }, + (DownwardRegistration::NoMatch, _) => (DownwardRegistration::NoMatch, no_match_reason), + (other, _) => (other, None), + } + } + + fn evaluate_reference_downward_registration_with_preferred_range( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + preferred_range: Option>, + allow_full_range_fallback: bool, + ) -> DownwardRegistration { + let config = OverlapSearchConfig::default(); + let max_overlap = previous.height().min(next.height()); + let max_motion_rows = max_directional_motion_rows(previous, next, config); + let mut candidates = preferred_range.as_ref().map_or_else(Vec::new, |range| { + collect_overlap_direction_matches( + previous, + next, + ScrollDirection::Down, + range.clone(), + config, + ) + }); + let mut no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; + + if candidates.is_empty() + && allow_full_range_fallback + && (motion_rows_hint.is_none() || self.transient_burst_search_enabled) + { + candidates = collect_overlap_direction_matches( + previous, + next, + ScrollDirection::Down, + 1..=max_motion_rows, + config, + ); + no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; + } + candidates.retain(|matched| { + downward_registration_has_meaningful_overlap(*matched, max_overlap, config) + }); + if candidates.is_empty() { + no_match_reason.get_or_insert("insufficient_overlap"); + } + + let classification = classify_downward_registration_candidates(&candidates); + let upward_veto = self.evaluate_reference_overlap_direction( + previous, + next, + ScrollDirection::Up, + motion_rows_hint, + ); + + match (classification, upward_veto) { + (DownwardRegistration::Matched(down), Some(up)) + if up.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + <= down.mean_abs_diff_x100 => + { + DownwardRegistration::NoMatch + }, + (DownwardRegistration::NoMatch, _) => { + let _ = no_match_reason; + DownwardRegistration::NoMatch + }, + (other, _) => other, + } + } + + fn sequential_downward_motion_ranges( + &self, + previous: &RgbaImage, + next: &RgbaImage, + config: OverlapSearchConfig, + ) -> Vec> { + let mut ranges = Vec::new(); + let local_motion_rows_hint = self.last_motion_rows_hint; + if let Some(local_range) = self.preferred_local_downward_motion_range_from_hint( + previous, + next, + local_motion_rows_hint, + config, + ) { + ranges.push(local_range); + } + if self.initial_downward_bootstrap_active() && self.last_motion_rows_hint.is_none() { + return ranges; + } + if let Some(transient_range) = self.transient_downward_motion_range(previous, next, config) + && !ranges.contains(&transient_range) + { + ranges.push(transient_range); + } + + ranges + } + + fn clear_last_downward_sample_registration(&mut self) { + self.last_downward_sample_registration_result = None; + self.last_downward_sample_registration_source = None; + self.last_downward_sample_registration_motion_rows = None; + self.last_downward_sample_registration_provisional_viewport_top_y = None; + self.last_observed_sample_registration_result = None; + self.last_observed_sample_registration_reason = None; + self.last_observed_sample_registration_motion_rows = None; + self.last_observed_sample_registration_mean_abs_diff_x100 = None; + self.last_preview_only_local_registration_result = None; + self.last_preview_only_local_registration_reason = None; + self.last_preview_only_local_registration_motion_rows = None; + self.last_preview_only_local_registration_mean_abs_diff_x100 = None; + self.last_downward_viewport_candidate_count = None; + self.last_downward_viewport_candidates_before_prune = None; + self.last_downward_viewport_candidates_after_prune = None; + self.blocked_underconsumed_observed_recovery_in_burst = false; + self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst = false; + self.blocked_followup_after_suppressed_huge_preview_local_jump = false; + self.blocked_followup_after_extreme_preview_local_tail = false; + self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = false; + } + + fn record_last_downward_sample_registration( + &mut self, + result: &'static str, + source: Option, + motion_rows: Option, + ) { + self.last_downward_sample_registration_result = Some(result); + self.last_downward_sample_registration_source = + source.map(DownwardSampleMatchSource::label); + self.last_downward_sample_registration_motion_rows = motion_rows; + } + + fn record_last_sample_eval_context(&mut self) { + self.last_sample_eval_last_motion_rows_hint = self.last_motion_rows_hint; + self.last_sample_eval_transient_motion_rows_hint = self.transient_motion_rows_hint; + self.last_sample_eval_effective_motion_rows_hint = self.effective_motion_rows_hint(); + self.last_sample_eval_transient_burst_search_enabled = self.transient_burst_search_enabled; + } + + fn transient_downward_motion_range( + &self, + previous: &RgbaImage, + next: &RgbaImage, + config: OverlapSearchConfig, + ) -> Option> { + let transient_motion_rows_hint = self.normalized_transient_motion_rows_hint()?; + let max_motion_rows = max_directional_motion_rows(previous, next, config); + + if transient_motion_rows_hint == 0 || transient_motion_rows_hint > max_motion_rows { + return None; + } + + let tolerance = (transient_motion_rows_hint / 2) + .clamp( + LOCAL_DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS, + LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS, + ) + .min(max_motion_rows); + let min_motion_rows = transient_motion_rows_hint.saturating_sub(tolerance).max(1); + let max_motion_rows = + transient_motion_rows_hint.saturating_add(tolerance).min(max_motion_rows); + + Some(min_motion_rows..=max_motion_rows) + } + + fn preferred_local_downward_motion_range_from_hint( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + config: OverlapSearchConfig, + ) -> Option> { + let max_motion_rows = max_directional_motion_rows(previous, next, config); + + if let Some(last_growth_rows) = motion_rows_hint { + let tolerance = (last_growth_rows / 2) + .clamp( + LOCAL_DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS, + LOCAL_DOWNWARD_SEARCH_MAX_TOLERANCE_ROWS, + ) + .min(max_motion_rows); + let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); + let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + + return Some(min_motion_rows..=max_motion_rows); + } + + Some(1..=INITIAL_DOWNWARD_MAX_MOTION_ROWS.min(max_motion_rows).max(1)) + } + + fn diagnose_reference_overlap_direction( + &self, + previous: &RgbaImage, + next: &RgbaImage, + direction: ScrollDirection, + motion_rows_hint: Option, + ) -> DirectionMatchEval { + let config = OverlapSearchConfig::default(); + let preferred_range = self + .preferred_motion_range_from_hint(previous, next, motion_rows_hint, config) + .map(OverlapSearchRange::from); + + self.diagnose_reference_overlap_direction_with_preferred_range( + previous, + next, + direction, + preferred_range, + false, + ) + } + + fn diagnose_reference_overlap_direction_with_preferred_range( + &self, + previous: &RgbaImage, + next: &RgbaImage, + direction: ScrollDirection, + preferred_range: Option, + allow_downward_full_range_fallback: bool, + ) -> DirectionMatchEval { + let config = OverlapSearchConfig::default(); + let max_motion_rows = max_directional_motion_rows(previous, next, config); + let preferred_only_match = preferred_range.and_then(|range| { + evaluate_overlap_direction(previous, next, direction, range.as_range(), config) + }); + let mut final_match = preferred_only_match; + let mut used_full_range_fallback = false; + + if final_match.is_none() && allow_downward_full_range_fallback { + final_match = + evaluate_overlap_direction(previous, next, direction, 1..=max_motion_rows, config); + used_full_range_fallback = final_match.is_some(); + } + + DirectionMatchEval { + preferred_range, + max_motion_rows, + preferred_only_match, + final_match, + used_full_range_fallback, + } + } + + fn evaluate_reference_overlap_direction_preferred_only( + &self, + previous: &RgbaImage, + next: &RgbaImage, + direction: ScrollDirection, + motion_rows_hint: Option, + ) -> Option { + let config = OverlapSearchConfig::default(); + let preferred_range = + self.preferred_motion_range_from_hint(previous, next, motion_rows_hint, config)?; + + evaluate_overlap_direction(previous, next, direction, preferred_range, config) + } + + fn preferred_motion_range_from_hint( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + config: OverlapSearchConfig, + ) -> Option> { + let max_motion_rows = max_directional_motion_rows(previous, next, config); + + if let Some(last_growth_rows) = motion_rows_hint { + let tolerance = DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS.min(max_motion_rows); + let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); + let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + + return Some(min_motion_rows..=max_motion_rows); + } + + Some(1..=INITIAL_DOWNWARD_MAX_MOTION_ROWS.min(max_motion_rows).max(1)) + } + + fn preferred_downward_motion_range_from_hint( + &self, + previous: &RgbaImage, + next: &RgbaImage, + motion_rows_hint: Option, + config: OverlapSearchConfig, + ) -> Option> { + let max_motion_rows = max_directional_motion_rows(previous, next, config); + + if let Some(last_growth_rows) = motion_rows_hint { + let tolerance = (last_growth_rows / 2) + .clamp( + DOWNWARD_KEYFRAME_SEARCH_MOTION_TOLERANCE_ROWS, + DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS, + ) + .min(max_motion_rows); + let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); + let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + + return Some(min_motion_rows..=max_motion_rows); + } + + Some(1..=INITIAL_DOWNWARD_MAX_MOTION_ROWS.min(max_motion_rows).max(1)) + } + + fn resolve_downward_viewport_candidate( + &mut self, + frame: &RgbaImage, + observed_match: DownwardSampleMatch, + ) -> DownwardViewportResolution { + let pending_suppressed_huge_preview_only_local_followup = + self.pending_suppressed_huge_preview_only_local_followup.take(); + let pending_suppressed_huge_preview_only_local_followup_remaining_blocks = + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks; + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 0; + let pending_extreme_preview_only_local_tail_followup = + self.pending_extreme_preview_only_local_tail_followup.take(); + let pending_extreme_preview_only_local_tail_followup_remaining_blocks = + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks; + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 0; + let mut candidates = Vec::with_capacity(DOWNWARD_KEYFRAME_SEARCH_LIMIT.saturating_add(1)); + let mut suppressed_observed_candidate = None; + let mut suppressed_preview_only_local_candidate = None; + + let provisional_viewport_top_y = + self.provisional_viewport_top_y_for_downward_sample_match(observed_match); + self.last_downward_sample_registration_provisional_viewport_top_y = + provisional_viewport_top_y; + + if let Some(viewport_top_y) = provisional_viewport_top_y { + let candidate = DownwardViewportCandidate { + source: observed_match.source.into(), + viewport_top_y, + motion_rows: observed_match.matched.motion_rows, + mean_abs_diff_x100: observed_match.matched.mean_abs_diff_x100, + }; + let suppress_observed = self.should_suppress_observed_sample_candidate(candidate); + let suppress_preview_local = + self.should_suppress_preview_only_local_candidate(candidate); + + if !suppress_observed && !suppress_preview_local { + candidates.push(candidate); + } else if suppress_observed + && candidate.source == DownwardViewportCandidateSource::ObservedSample + { + suppressed_observed_candidate = Some(candidate); + } else if suppress_preview_local + && candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + { + suppressed_preview_only_local_candidate = Some(candidate); + } + } + self.collect_committed_downward_viewport_candidates(frame, &mut candidates); + if self + .should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( + suppressed_preview_only_local_candidate, + &candidates, + ) { + self.pending_suppressed_huge_preview_only_local_followup = + suppressed_preview_only_local_candidate; + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = self + .suppressed_huge_preview_only_local_followup_block_budget( + suppressed_preview_only_local_candidate, + ); + candidates.clear(); + } + if self.should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + pending_suppressed_huge_preview_only_local_followup, + &candidates, + ) { + if let Some(pending_candidate) = pending_suppressed_huge_preview_only_local_followup { + if pending_suppressed_huge_preview_only_local_followup_remaining_blocks > 1 { + self.pending_suppressed_huge_preview_only_local_followup = + Some(pending_candidate); + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = + pending_suppressed_huge_preview_only_local_followup_remaining_blocks - 1; + } + } + self.blocked_followup_after_suppressed_huge_preview_local_jump = true; + candidates.clear(); + } + if self.should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + pending_extreme_preview_only_local_tail_followup, + &candidates, + ) { + if let Some(pending_candidate) = pending_extreme_preview_only_local_tail_followup + && pending_extreme_preview_only_local_tail_followup_remaining_blocks > 1 + { + self.pending_extreme_preview_only_local_tail_followup = Some(pending_candidate); + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = + pending_extreme_preview_only_local_tail_followup_remaining_blocks - 1; + } + self.blocked_followup_after_extreme_preview_local_tail = true; + candidates.clear(); + } + self.restore_corroborated_observed_candidate( + suppressed_observed_candidate, + &mut candidates, + ); + let preview_only_local_candidate_before_prune = + candidates.iter().copied().find(|candidate| { + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + }); + let candidates_before_prune = candidates.clone(); + self.last_downward_viewport_candidates_before_prune = + Some(format_downward_viewport_candidates(&candidates)); + self.prune_committed_keyframe_candidates_outside_local_continuity(&mut candidates); + self.restore_repeated_small_preview_only_local_candidate_after_empty_prune( + preview_only_local_candidate_before_prune, + &mut candidates, + ); + if self.should_fail_closed_lagging_exactly_corroborated_preview_local_tail_in_burst( + &candidates, + ) { + self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst = true; + candidates.clear(); + } + if self.should_fail_closed_underconsumed_observed_recovery_in_burst( + &candidates_before_prune, + &candidates, + ) { + self.blocked_underconsumed_observed_recovery_in_burst = true; + candidates.clear(); + } + self.last_downward_viewport_candidate_count = Some(candidates.len()); + self.last_downward_viewport_candidates_after_prune = + Some(format_downward_viewport_candidates(&candidates)); + select_downward_viewport_candidate(&mut candidates) + } + + fn should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( + &self, + suppressed_preview_only_local_candidate: Option, + committed_candidates: &[DownwardViewportCandidate], + ) -> bool { + let Some(candidate) = suppressed_preview_only_local_candidate else { + return false; + }; + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { + return false; + } + + let large_far_recovery_threshold = last_motion_rows_hint + .saturating_mul(3) + .max(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_mul(2)); + + self.transient_burst_search_enabled + && self.last_observed_sample_registration_result == Some("matched") + && self.last_observed_sample_registration_motion_rows == Some(candidate.motion_rows) + && candidate.motion_rows > large_far_recovery_threshold + && self.growth_history.last().is_some_and(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && commit.growth_rows + >= last_motion_rows_hint + .saturating_sub(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + }) && committed_candidates.iter().any(|committed| { + committed.source == DownwardViewportCandidateSource::CommittedKeyframe + && committed.motion_rows == candidate.motion_rows + && committed.viewport_top_y == candidate.viewport_top_y + }) + } + + fn should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + &self, + pending_suppressed_preview_only_local_candidate: Option, + candidates: &[DownwardViewportCandidate], + ) -> bool { + let Some(pending_candidate) = pending_suppressed_preview_only_local_candidate else { + return false; + }; + if pending_candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { + return false; + } + + self.transient_burst_search_enabled + && self.last_preview_only_local_registration_result == Some("no_match") + && self.last_observed_sample_registration_result == Some("matched") + && self.last_observed_sample_registration_motion_rows + == Some(pending_candidate.motion_rows) + && candidates.iter().all(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + }) && candidates.iter().any(|candidate| { + candidate.viewport_top_y == pending_candidate.viewport_top_y + && candidate.motion_rows == pending_candidate.motion_rows + }) + } + + fn should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + &self, + pending_preview_only_local_candidate: Option, + candidates: &[DownwardViewportCandidate], + ) -> bool { + let Some(pending_candidate) = pending_preview_only_local_candidate else { + return false; + }; + if pending_candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { + return false; + } + + self.transient_burst_search_enabled + && candidates.iter().all(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + }) && candidates.iter().any(|candidate| { + candidate.viewport_top_y == pending_candidate.viewport_top_y + && candidate.motion_rows == pending_candidate.motion_rows + }) + } + + fn suppressed_huge_preview_only_local_followup_block_budget( + &self, + candidate: Option, + ) -> u8 { + let Some(candidate) = candidate else { + return 3; + }; + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return 3; + }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { + return 3; + } + + let continuity_rows = last_motion_rows_hint.max(1); + let far_recovery_ratio = + candidate.motion_rows.saturating_add(continuity_rows.saturating_sub(1)) + / continuity_rows; + + u8::try_from(far_recovery_ratio.clamp(3, 5)).unwrap_or(5) + } + + fn restore_corroborated_observed_candidate( + &self, + suppressed_observed_candidate: Option, + candidates: &mut Vec, + ) { + let Some(candidate) = suppressed_observed_candidate else { + return; + }; + if !self.observed_candidate_can_recover_from_committed_corroboration(candidate) { + return; + } + if candidates.iter().any(|other| { + other.source == DownwardViewportCandidateSource::CommittedKeyframe + && other.viewport_top_y == candidate.viewport_top_y + && other.motion_rows == candidate.motion_rows + }) { + candidates.push(candidate); + } + } + + fn observed_candidate_can_recover_from_committed_corroboration( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if candidate.source != DownwardViewportCandidateSource::ObservedSample { + return false; + } + + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let corroboration_cap = + last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + + self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y) <= corroboration_cap + } + + fn restore_repeated_small_preview_only_local_candidate_after_empty_prune( + &mut self, + preview_only_local_candidate_before_prune: Option, + candidates_after_prune: &mut Vec, + ) { + let Some(candidate) = preview_only_local_candidate_before_prune else { + self.last_blocked_preview_only_local_candidate = None; + return; + }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample + || !candidates_after_prune.is_empty() + || !self.repeated_preview_only_local_candidate_can_restore_after_empty_prune(candidate) + { + self.last_blocked_preview_only_local_candidate = None; + return; + } + + let repeats = match self.last_blocked_preview_only_local_candidate { + Some(previous) if previous.candidate == candidate => previous.repeats.saturating_add(1), + _ => 1, + }; + self.last_blocked_preview_only_local_candidate = + Some(BlockedPreviewOnlyLocalCandidate { candidate, repeats }); + + if repeats >= 2 { + candidates_after_prune.push(candidate); + self.last_blocked_preview_only_local_candidate = None; + } + } + + fn repeated_preview_only_local_candidate_can_restore_after_empty_prune( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.motion_rows <= REPEATED_PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && self.transient_burst_search_enabled + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self.last_motion_rows_hint.is_some() + } + + fn should_fail_closed_lagging_exactly_corroborated_preview_local_tail_in_burst( + &self, + candidates_after_prune: &[DownwardViewportCandidate], + ) -> bool { + if !self.transient_burst_search_enabled { + return false; + } + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(preview_only_local_candidate) = + candidates_after_prune.iter().copied().find(|candidate| { + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + }) + else { + return false; + }; + + preview_only_local_candidate.motion_rows + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_div(2) + && preview_only_local_candidate.motion_rows + < last_motion_rows_hint + .saturating_sub(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + && candidates_after_prune.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.viewport_top_y == preview_only_local_candidate.viewport_top_y + && candidate.motion_rows == preview_only_local_candidate.motion_rows + && candidate.mean_abs_diff_x100 + <= preview_only_local_candidate + .mean_abs_diff_x100 + .saturating_add(DIRECTION_WARNING_MARGIN_X100) + }) + } + + fn should_fail_closed_underconsumed_observed_recovery_in_burst( + &self, + candidates_before_prune: &[DownwardViewportCandidate], + candidates_after_prune: &[DownwardViewportCandidate], + ) -> bool { + let Some(observed_candidate) = candidates_after_prune + .iter() + .copied() + .find(|candidate| candidate.source == DownwardViewportCandidateSource::ObservedSample) + else { + return false; + }; + + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + + if self.last_preview_only_downward_local_sample.is_some() + || !self + .transient_burst_motion_hint_exceeds_local_authority(observed_candidate.motion_rows) + || last_motion_rows_hint + < observed_candidate + .motion_rows + .saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + { + return false; + } + + let has_same_motion_committed_corroboration = + candidates_after_prune.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.viewport_top_y == observed_candidate.viewport_top_y + && candidate.motion_rows == observed_candidate.motion_rows + }); + if !has_same_motion_committed_corroboration { + return false; + } + + candidates_before_prune.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.motion_rows > observed_candidate.motion_rows + && candidate.motion_rows >= last_motion_rows_hint + && candidate.viewport_top_y >= observed_candidate.viewport_top_y + && candidate.viewport_top_y.abs_diff(observed_candidate.viewport_top_y) + <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && candidate.mean_abs_diff_x100 + <= observed_candidate + .mean_abs_diff_x100 + .saturating_add(DIRECTION_WARNING_MARGIN_X100) + }) + } + + fn prune_committed_keyframe_candidates_outside_local_continuity( + &mut self, + candidates: &mut Vec, + ) { + let has_committed_candidate = candidates.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + }); + let mut local_anchor = best_local_downward_viewport_candidate(candidates); + if local_anchor.is_some_and(|anchor| { + has_committed_candidate + && anchor.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && self.transient_burst_motion_hint_exceeds_local_authority(anchor.motion_rows) + && !self + .preview_only_local_anchor_has_exact_committed_corroboration(anchor, candidates) + && !self.preview_only_local_candidate_has_material_progress(anchor) + && ((anchor.motion_rows <= TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS + && self.consecutive_transient_burst_missing_downward_candidate_frames < 2) + || candidates.iter().any(|candidate| { + self.committed_candidate_can_plausibly_replace_underconsumed_preview_local_anchor( + anchor, + *candidate, + ) + })) + }) { + candidates.retain(|candidate| { + candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample + }); + local_anchor = best_local_downward_viewport_candidate(candidates); + } + + let Some(local_anchor) = local_anchor else { + candidates.retain(|candidate| { + candidate.source != DownwardViewportCandidateSource::CommittedKeyframe + || !self.transient_burst_search_enabled + || !self.fallback_downward_growth_exceeds_continuity_budget( + candidate.viewport_top_y, + ) || self.transient_burst_growth_matches_pending_hint_band(candidate.viewport_top_y) + || self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y) + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + }); + + if let Some(max_bootstrap_growth_rows) = + self.bootstrap_committed_keyframe_growth_cap_rows() + { + candidates.retain(|candidate| { + candidate.source != DownwardViewportCandidateSource::CommittedKeyframe + || self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y) + <= max_bootstrap_growth_rows + }); + } + self.prune_committed_keyframe_candidates_without_local_anchor(candidates); + return; + }; + let allowed_overrun_rows = self + .max_committed_keyframe_local_overrun_rows(local_anchor) + .max(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + let max_allowed_motion_rows = self.max_committed_keyframe_motion_rows(local_anchor); + let max_allowed_viewport_top_y = local_anchor + .viewport_top_y + .saturating_add(i32::try_from(allowed_overrun_rows).unwrap_or(i32::MAX)); + let local_observed_has_same_motion_committed_corroboration = local_anchor.source + == DownwardViewportCandidateSource::ObservedSample + && candidates.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.viewport_top_y == local_anchor.viewport_top_y + && candidate.motion_rows == local_anchor.motion_rows + }); + + candidates.retain(|candidate| { + candidate.source != DownwardViewportCandidateSource::CommittedKeyframe + || (candidate.viewport_top_y <= max_allowed_viewport_top_y + && candidate.motion_rows <= max_allowed_motion_rows) + || (!local_observed_has_same_motion_committed_corroboration + && self.committed_candidate_can_override_untrustworthy_observed_local_recovery( + local_anchor, + *candidate, + )) + }); + self.prune_committed_keyframe_candidates_for_transient_burst(candidates); + } + + fn preview_only_local_anchor_has_exact_committed_corroboration( + &self, + local_anchor: DownwardViewportCandidate, + candidates: &[DownwardViewportCandidate], + ) -> bool { + local_anchor.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && candidates.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.viewport_top_y == local_anchor.viewport_top_y + && candidate.motion_rows == local_anchor.motion_rows + && candidate.mean_abs_diff_x100 + <= local_anchor + .mean_abs_diff_x100 + .saturating_add(DIRECTION_WARNING_MARGIN_X100) + }) + } + + fn prune_committed_keyframe_candidates_without_local_anchor( + &mut self, + candidates: &mut Vec, + ) { + if !candidates + .iter() + .all(|candidate| candidate.source == DownwardViewportCandidateSource::CommittedKeyframe) + { + return; + } + + let Some(preferred) = candidates.iter().copied().min_by(|left, right| { + left.motion_rows + .cmp(&right.motion_rows) + .then(left.mean_abs_diff_x100.cmp(&right.mean_abs_diff_x100)) + .then(left.viewport_top_y.cmp(&right.viewport_top_y)) + }) else { + return; + }; + if self.should_fail_closed_far_committed_only_recovery_without_local_anchor( + preferred, candidates, + ) { + if self + .should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + preferred, + self.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y), + ) { + self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; + } + candidates.clear(); + return; + } + + candidates.retain(|candidate| *candidate == preferred); + } + + fn should_fail_closed_far_committed_only_recovery_without_local_anchor( + &self, + preferred: DownwardViewportCandidate, + candidates: &[DownwardViewportCandidate], + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + if !self.transient_burst_search_enabled { + return false; + } + let preferred_growth_rows = + self.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y); + if self + .should_fail_closed_underconsumed_committed_only_recovery_after_suppressed_preview_local_match( + preferred, + preferred_growth_rows, + ) { + return true; + } + if self + .should_fail_closed_committed_only_recovery_after_corroborated_sample_registration_without_viewport_anchor( + preferred, + preferred_growth_rows, + ) + { + return true; + } + if self + .should_fail_closed_committed_only_recovery_when_observed_burst_outpaces_recent_preview_local_commit( + preferred, + preferred_growth_rows, + ) + { + return true; + } + if self.should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + preferred, + preferred_growth_rows, + ) { + return true; + } + if self.last_preview_only_downward_local_sample.is_some() + && self.last_preview_only_local_registration_result == Some("matched") + && last_motion_rows_hint <= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && self.last_preview_only_local_registration_motion_rows.is_some_and( + |local_motion_rows| { + local_motion_rows + <= last_motion_rows_hint + .saturating_add(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + && preferred_growth_rows + > local_motion_rows.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + }, + ) { + return true; + } + if last_motion_rows_hint > DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS.saturating_mul(2) { + let all_candidates_low_confidence = candidates.iter().all(|candidate| { + candidate.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(4) + }); + + return preferred_growth_rows <= last_motion_rows_hint && all_candidates_low_confidence; + } + + let far_growth_threshold = PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + .max(last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS)); + + self.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y) + > far_growth_threshold + && candidates.iter().all(|candidate| { + self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y) + > far_growth_threshold + }) + } + + fn should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + &self, + preferred: DownwardViewportCandidate, + preferred_growth_rows: u32, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + if preferred.source != DownwardViewportCandidateSource::CommittedKeyframe { + return false; + } + + let large_far_recovery_threshold = last_motion_rows_hint + .saturating_mul(3) + .max(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_mul(2)); + let observed_material_lag_threshold = PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + .max(last_motion_rows_hint.saturating_mul(2)); + let observed_corroborates_or_materially_lags = + self.last_observed_sample_registration_result == Some("matched") + && self.last_observed_sample_registration_motion_rows.is_some_and( + |observed_motion_rows| { + observed_motion_rows == preferred.motion_rows + || observed_motion_rows.saturating_add(observed_material_lag_threshold) + < preferred.motion_rows + }, + ); + + self.transient_burst_search_enabled + && self.last_preview_only_local_registration_result == Some("matched") + && self.last_preview_only_local_registration_motion_rows == Some(preferred.motion_rows) + && observed_corroborates_or_materially_lags + && preferred.motion_rows > large_far_recovery_threshold + && preferred_growth_rows > large_far_recovery_threshold + && self.growth_history.last().is_some_and(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && commit.growth_rows + >= last_motion_rows_hint + .saturating_sub(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + }) + } + + fn should_fail_closed_underconsumed_committed_only_recovery_after_suppressed_preview_local_match( + &self, + preferred: DownwardViewportCandidate, + preferred_growth_rows: u32, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(local_motion_rows) = self.last_preview_only_local_registration_motion_rows else { + return false; + }; + + self.last_preview_only_downward_local_sample.is_some() + && self.last_preview_only_local_registration_result == Some("matched") + && self.transient_burst_motion_hint_exceeds_local_authority(preferred.motion_rows) + && !self.transient_burst_growth_matches_pending_hint_band(preferred.viewport_top_y) + && local_motion_rows > PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && local_motion_rows + > preferred + .motion_rows + .saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + && preferred_growth_rows + <= last_motion_rows_hint + .saturating_mul(2) + .max(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS) + } + + fn should_fail_closed_committed_only_recovery_after_corroborated_sample_registration_without_viewport_anchor( + &self, + preferred: DownwardViewportCandidate, + preferred_growth_rows: u32, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(observed_motion_rows) = self.last_observed_sample_registration_motion_rows else { + return false; + }; + let Some(local_motion_rows) = self.last_preview_only_local_registration_motion_rows else { + return false; + }; + + let corroborated_motion_floor = + last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + let corroborated_motion_ceiling = observed_motion_rows + .max(local_motion_rows) + .saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + + preferred.source == DownwardViewportCandidateSource::CommittedKeyframe + && self.transient_burst_search_enabled + && self.last_preview_only_downward_local_sample.is_some() + && self.last_observed_sample_registration_result == Some("matched") + && self.last_preview_only_local_registration_result == Some("matched") + && last_motion_rows_hint <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && observed_motion_rows > corroborated_motion_floor + && local_motion_rows > corroborated_motion_floor + && preferred_growth_rows > corroborated_motion_floor + && preferred.motion_rows >= local_motion_rows + && preferred_growth_rows <= corroborated_motion_ceiling + } + + fn should_fail_closed_committed_only_recovery_when_observed_burst_outpaces_recent_preview_local_commit( + &self, + preferred: DownwardViewportCandidate, + preferred_growth_rows: u32, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(observed_motion_rows) = self.last_observed_sample_registration_motion_rows else { + return false; + }; + + let recent_preview_local_commit = self.growth_history.last().is_some_and(|commit| { + commit.decision_source + == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && commit.growth_rows + >= last_motion_rows_hint.saturating_sub(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + }); + let corroborated_motion_floor = + last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + + preferred.source == DownwardViewportCandidateSource::CommittedKeyframe + && self.transient_burst_search_enabled + && recent_preview_local_commit + && self.last_observed_sample_registration_result == Some("matched") + && self.last_preview_only_local_registration_result == Some("no_match") + && last_motion_rows_hint <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && observed_motion_rows > corroborated_motion_floor + && preferred_growth_rows > corroborated_motion_floor + && preferred.motion_rows.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + < observed_motion_rows + } + + fn should_suppress_preview_only_local_candidate( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && candidate.motion_rows > PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && !self.preview_only_local_candidate_remains_trustworthy_in_burst(candidate) + } + + fn should_suppress_observed_sample_candidate( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.source == DownwardViewportCandidateSource::ObservedSample + && self.transient_burst_search_enabled + && self.fallback_downward_growth_exceeds_continuity_budget(candidate.viewport_top_y) + && !self.observed_sample_candidate_remains_trustworthy_in_burst(candidate) + } + + fn observed_sample_candidate_remains_trustworthy_in_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if candidate.source != DownwardViewportCandidateSource::ObservedSample { + return false; + } + + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) + && self.last_motion_rows_hint.is_some_and(|last_hint| { + candidate.motion_rows.saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + >= last_hint && candidate.motion_rows <= last_hint + }) && candidate.mean_abs_diff_x100 <= DIRECTION_WARNING_MARGIN_X100.saturating_mul(6) + && self.transient_pending_growth_cap_rows().is_some_and(|cap| growth_rows <= cap) + } + + fn preview_only_local_candidate_has_material_progress( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if self.seeded_preview_only_local_catch_up_candidate_can_commit(candidate) { + return true; + } + + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample && { + let growth_rows = + self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + growth_rows >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + || self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && growth_rows.saturating_add(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + >= last_hint + }) || self.last_motion_rows_hint.is_some_and(|last_hint| { + last_hint >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && candidate.motion_rows.abs_diff(last_hint) + <= PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS + }) + } + } + + fn preview_only_local_candidate_remains_trustworthy_in_burst( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { + return true; + } + if candidate.motion_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS { + return true; + } + + self.transient_burst_growth_matches_pending_hint_band(candidate.viewport_top_y) + && self.last_motion_rows_hint.is_some_and(|last_hint| { + candidate.motion_rows + <= last_hint.saturating_add(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS) + }) + } + + fn seeded_preview_only_local_catch_up_candidate_can_commit( + &self, + candidate: DownwardViewportCandidate, + ) -> bool { + candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample + && self.seeded_preview_only_local_after_observed_burst_commit + && candidate.viewport_top_y > self.current_viewport_top_y + && candidate.motion_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + } + + fn prune_committed_keyframe_candidates_for_transient_burst( + &mut self, + candidates: &mut Vec, + ) { + if !self.transient_burst_search_enabled { + return; + } + + let Some(local_candidate) = candidates + .iter() + .copied() + .filter(|candidate| candidate.source == DownwardViewportCandidateSource::ObservedSample) + .min_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.motion_rows.cmp(&right.motion_rows)) + }) + else { + return; + }; + + let Some(previous_growth_rows) = self.last_motion_rows_hint else { + return; + }; + + if local_candidate.motion_rows <= previous_growth_rows { + return; + } + + candidates.retain(|candidate| { + candidate.source != DownwardViewportCandidateSource::CommittedKeyframe + || candidate.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + < local_candidate.mean_abs_diff_x100 + }); + } + + fn max_committed_keyframe_local_overrun_rows( + &self, + local_anchor: DownwardViewportCandidate, + ) -> u32 { + self.max_committed_keyframe_motion_rows(local_anchor).clamp( + DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS, + DOWNWARD_COMMITTED_KEYFRAME_LOCAL_OVERRUN_MAX_ROWS, + ) + } + + fn max_committed_keyframe_motion_rows(&self, local_anchor: DownwardViewportCandidate) -> u32 { + let continuity_rows = self + .last_motion_rows_hint + .unwrap_or(local_anchor.motion_rows) + .max(local_anchor.motion_rows); + let tolerance_rows = (continuity_rows / 2).clamp(1, DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + + continuity_rows.saturating_add(tolerance_rows) } - fn diagnose_reference_overlap_direction( + fn committed_candidate_can_plausibly_replace_underconsumed_preview_local_anchor( &self, - previous: &RgbaImage, - next: &RgbaImage, - direction: ScrollDirection, - motion_rows_hint: Option, - ) -> DirectionMatchEval { - let config = OverlapSearchConfig::default(); - let preferred_range = self - .preferred_motion_range_from_hint(previous, next, motion_rows_hint, config) - .map(OverlapSearchRange::from); + local_anchor: DownwardViewportCandidate, + committed_candidate: DownwardViewportCandidate, + ) -> bool { + if committed_candidate.source != DownwardViewportCandidateSource::CommittedKeyframe { + return false; + } - self.diagnose_reference_overlap_direction_with_preferred_range( - previous, - next, - direction, - preferred_range, - false, + let allowed_overrun_rows = self + .max_committed_keyframe_local_overrun_rows(local_anchor) + .max(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + let max_allowed_motion_rows = self.max_committed_keyframe_motion_rows(local_anchor); + let max_allowed_viewport_top_y = local_anchor + .viewport_top_y + .saturating_add(i32::try_from(allowed_overrun_rows).unwrap_or(i32::MAX)); + let local_anchor_tracks_recent_continuity = self + .last_motion_rows_hint + .is_some_and(|last_hint| local_anchor.motion_rows >= last_hint); + let committed_is_not_materially_worse_than_local_anchor = committed_candidate + .mean_abs_diff_x100 + <= local_anchor.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100); + + (committed_candidate.viewport_top_y <= max_allowed_viewport_top_y + && committed_candidate.motion_rows <= max_allowed_motion_rows) + && (!local_anchor_tracks_recent_continuity + || committed_is_not_materially_worse_than_local_anchor) + || self.transient_burst_growth_matches_pending_hint_band( + committed_candidate.viewport_top_y, + ) || self.committed_candidate_can_override_untrustworthy_observed_local_recovery( + local_anchor, + committed_candidate, ) } - fn diagnose_reference_overlap_direction_with_preferred_range( + fn committed_candidate_can_override_untrustworthy_observed_local_recovery( &self, - previous: &RgbaImage, - next: &RgbaImage, - direction: ScrollDirection, - preferred_range: Option, - allow_downward_full_range_fallback: bool, - ) -> DirectionMatchEval { - let config = OverlapSearchConfig::default(); - let max_motion_rows = max_directional_motion_rows(previous, next, config); - let preferred_only_match = preferred_range.and_then(|range| { - evaluate_overlap_direction(previous, next, direction, range.as_range(), config) - }); - let mut final_match = preferred_only_match; - let mut used_full_range_fallback = false; + local_anchor: DownwardViewportCandidate, + committed_candidate: DownwardViewportCandidate, + ) -> bool { + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; + let Some(transient_growth_cap_rows) = self.transient_pending_growth_cap_rows() else { + return false; + }; + if committed_candidate.source != DownwardViewportCandidateSource::CommittedKeyframe { + return false; + } + let local_growth_rows = + self.growth_rows_for_candidate_viewport_top_y(local_anchor.viewport_top_y); + let committed_growth_rows = + self.growth_rows_for_candidate_viewport_top_y(committed_candidate.viewport_top_y); + + local_anchor.source == DownwardViewportCandidateSource::ObservedSample + && self.transient_burst_motion_hint_exceeds_local_authority(local_anchor.motion_rows) + && local_anchor.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(4) + && local_anchor.motion_rows + <= last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS) + && (committed_growth_rows + <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS.saturating_mul(2) + || self.transient_burst_growth_matches_pending_hint_band( + committed_candidate.viewport_top_y, + )) && committed_candidate.mean_abs_diff_x100 + <= DIRECTION_WARNING_MARGIN_X100.saturating_mul(2) + && committed_candidate + .mean_abs_diff_x100 + .saturating_add(DIRECTION_WARNING_MARGIN_X100.saturating_mul(3)) + < local_anchor.mean_abs_diff_x100 + && committed_candidate.motion_rows + > local_anchor + .motion_rows + .saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + && committed_growth_rows + > local_growth_rows.saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) + && committed_growth_rows <= transient_growth_cap_rows + } + + fn bootstrap_committed_keyframe_growth_cap_rows(&self) -> Option { + if !self.initial_downward_bootstrap_active() { + return None; + } - if final_match.is_none() && allow_downward_full_range_fallback { - final_match = - evaluate_overlap_direction(previous, next, direction, 1..=max_motion_rows, config); - used_full_range_fallback = final_match.is_some(); + self.transient_pending_growth_cap_rows() + } + + fn transient_pending_growth_cap_rows(&self) -> Option { + let hint = self.normalized_transient_motion_rows_hint()?; + let tolerance = (hint / 2).clamp(1, PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS); + + Some(hint.saturating_add(tolerance)) + } + + fn transient_burst_growth_matches_pending_hint_band( + &self, + candidate_viewport_top_y: i32, + ) -> bool { + if !self.transient_burst_search_enabled { + return false; } - DirectionMatchEval { - preferred_range, - max_motion_rows, - preferred_only_match, - final_match, - used_full_range_fallback, + let Some(transient_hint) = self.normalized_transient_motion_rows_hint() else { + return false; + }; + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + let min_growth_rows = + (transient_hint / 2).max(self.last_motion_rows_hint.unwrap_or_default()); + + self.transient_pending_growth_cap_rows() + .is_some_and(|cap| growth_rows >= min_growth_rows && growth_rows <= cap) + } + + fn collect_committed_downward_viewport_candidates( + &self, + frame: &RgbaImage, + candidates: &mut Vec, + ) { + self.collect_committed_downward_viewport_candidates_with_mode( + frame, + candidates, + CommittedDownwardViewportCandidateMode::IncludeRecentHistory, + ); + } + + fn collect_fallback_downward_viewport_candidates( + &self, + frame: &RgbaImage, + candidates: &mut Vec, + ) { + self.collect_committed_downward_viewport_candidates_with_mode( + frame, + candidates, + CommittedDownwardViewportCandidateMode::LastCommittedOnly, + ); + } + + fn collect_committed_downward_viewport_candidates_with_mode( + &self, + frame: &RgbaImage, + candidates: &mut Vec, + mode: CommittedDownwardViewportCandidateMode, + ) { + self.push_downward_viewport_candidate( + &self.last_committed_frame, + self.current_viewport_top_y, + frame, + DownwardViewportCandidateSource::CommittedKeyframe, + candidates, + ); + + if mode == CommittedDownwardViewportCandidateMode::LastCommittedOnly + || DOWNWARD_KEYFRAME_SEARCH_LIMIT <= 1 + { + return; + } + + for commit in self + .growth_history + .iter() + .rev() + .skip(1) + .take(DOWNWARD_KEYFRAME_SEARCH_LIMIT.saturating_sub(1)) + { + self.push_downward_viewport_candidate( + &commit.frame, + commit.viewport_top_y, + frame, + DownwardViewportCandidateSource::CommittedKeyframe, + candidates, + ); } } - fn evaluate_reference_overlap_direction_preferred_only( + fn push_downward_viewport_candidate( &self, - previous: &RgbaImage, - next: &RgbaImage, - direction: ScrollDirection, - motion_rows_hint: Option, - ) -> Option { - let config = OverlapSearchConfig::default(); - let preferred_range = - self.preferred_motion_range_from_hint(previous, next, motion_rows_hint, config)?; + reference: &RgbaImage, + reference_viewport_top_y: i32, + frame: &RgbaImage, + source: DownwardViewportCandidateSource, + candidates: &mut Vec, + ) { + let predicted_motion_rows = self.downward_keyframe_motion_hint(reference_viewport_top_y); + let allow_full_range_fallback = + !(self.initial_downward_bootstrap_active() && predicted_motion_rows.is_none()); + let mut registration = self.evaluate_reference_downward_registration( + reference, + frame, + predicted_motion_rows, + allow_full_range_fallback, + ); - evaluate_overlap_direction(previous, next, direction, preferred_range, config) + if source == DownwardViewportCandidateSource::CommittedKeyframe + && self.should_retry_committed_keyframe_registration_across_full_range(registration) + { + let full_range_registration = self + .evaluate_reference_downward_registration_with_preferred_range( + reference, + frame, + predicted_motion_rows, + None, + true, + ); + registration = self.prefer_full_range_committed_keyframe_registration( + registration, + full_range_registration, + ); + } + + if let DownwardRegistration::Matched(matched) = registration { + if self.bootstrap_motion_exceeds_pending_hint(matched.motion_rows) { + return; + } + + let max_overlap = reference.height().min(frame.height()); + let min_keyframe_overlap_rows = OverlapSearchConfig::default() + .min_overlap_rows + .max(max_overlap / DOWNWARD_KEYFRAME_MIN_OVERLAP_DIVISOR) + .max(1); + let overlap_rows = max_overlap.saturating_sub(matched.motion_rows); + + if overlap_rows < min_keyframe_overlap_rows { + return; + } + + let viewport_top_y = reference_viewport_top_y + .saturating_add(i32::try_from(matched.motion_rows).unwrap_or_default()); + + if viewport_top_y <= self.current_viewport_top_y { + return; + } + + candidates.push(DownwardViewportCandidate { + source, + viewport_top_y, + motion_rows: matched.motion_rows, + mean_abs_diff_x100: matched.mean_abs_diff_x100, + }); + } } - fn preferred_motion_range_from_hint( + fn should_retry_committed_keyframe_registration_across_full_range( &self, - previous: &RgbaImage, - next: &RgbaImage, - motion_rows_hint: Option, - config: OverlapSearchConfig, - ) -> Option> { - let max_motion_rows = max_directional_motion_rows(previous, next, config); + registration: DownwardRegistration, + ) -> bool { + let DownwardRegistration::Matched(matched) = registration else { + return false; + }; + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { + return false; + }; - if let Some(last_growth_rows) = motion_rows_hint { - let tolerance = DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS.min(max_motion_rows); - let min_motion_rows = last_growth_rows.saturating_sub(tolerance).max(1); - let max_motion_rows = last_growth_rows.saturating_add(tolerance).min(max_motion_rows); + let low_confidence_match = + matched.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(4); + let tiny_underconsumed_match = self + .transient_burst_motion_hint_exceeds_local_authority(matched.motion_rows) + && matched.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(4) + && matched.motion_rows + <= last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); + let large_overshot_match = matched.motion_rows > last_motion_rows_hint.saturating_mul(2); - return Some(min_motion_rows..=max_motion_rows); + low_confidence_match && (tiny_underconsumed_match || large_overshot_match) + } + + fn prefer_full_range_committed_keyframe_registration( + &self, + preferred_range_registration: DownwardRegistration, + full_range_registration: DownwardRegistration, + ) -> DownwardRegistration { + match (preferred_range_registration, full_range_registration) { + (DownwardRegistration::Matched(preferred), DownwardRegistration::Matched(full)) + if full.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + < preferred.mean_abs_diff_x100 + && preferred.motion_rows.abs_diff(full.motion_rows) + > UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS => + { + DownwardRegistration::Matched(full) + }, + (preferred, _) => preferred, } + } - Some(1..=INITIAL_DOWNWARD_MAX_MOTION_ROWS.min(max_motion_rows).max(1)) + fn downward_keyframe_motion_hint(&self, reference_viewport_top_y: i32) -> Option { + let last_motion_rows = self.last_motion_rows_hint?; + let already_traversed_rows = u32::try_from( + self.current_viewport_top_y.saturating_sub(reference_viewport_top_y).max(0), + ) + .unwrap_or_default(); + + Some(already_traversed_rows.saturating_add(last_motion_rows)) } fn fallback_downward_growth_blocked_while_resume_frontier_active( &mut self, + candidate_viewport_top_y: i32, motion_rows: u32, preview_changed: bool, decision_source: &'static str, - previous_sample_frame: RgbaImage, - previous_sample_fingerprint: Option>, ) -> Option { let resume_frontier_top_y = self.resume_frontier_top_y?; - let candidate_viewport_top_y = self - .current_viewport_top_y - .saturating_add(i32::try_from(motion_rows).unwrap_or_default()); let growth_rows = if candidate_viewport_top_y <= resume_frontier_top_y { 0 } else { @@ -1816,128 +4864,126 @@ impl ScrollSession { Some(growth_rows), Some(decision_source), ); - self.restore_last_sample(previous_sample_frame, previous_sample_fingerprint); Some(preview_update_outcome(preview_changed)) } + fn fallback_downward_growth_exceeds_continuity_budget( + &self, + candidate_viewport_top_y: i32, + ) -> bool { + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + let Some(base_continuity_rows) = self.last_motion_rows_hint else { + return false; + }; + let local_overrun_rows = base_continuity_rows + .saturating_mul(2) + .clamp(FALLBACK_DOWNWARD_GROWTH_MIN_ROWS, FALLBACK_DOWNWARD_GROWTH_MAX_ROWS); + let preview_local_rows = self + .last_preview_only_downward_local_sample + .as_ref() + .map(|sample| { + u32::try_from( + sample.viewport_top_y.saturating_sub(self.current_viewport_top_y).max(0), + ) + .unwrap_or_default() + }) + .unwrap_or_default(); + let max_growth_rows = preview_local_rows.saturating_add(local_overrun_rows); + + growth_rows > max_growth_rows + } + fn observe_fallback_downward_growth( &mut self, frame: RgbaImage, preview_changed: bool, - previous_sample_frame: RgbaImage, - previous_sample_fingerprint: Option>, ) -> Result { - let down_match = self.evaluate_reference_overlap_direction( - &self.last_committed_frame, - &frame, - ScrollDirection::Down, - self.last_motion_rows_hint, - ); - let up_match = self.evaluate_reference_overlap_direction( - &self.last_committed_frame, - &frame, - ScrollDirection::Up, - self.last_motion_rows_hint, - ); + let mut candidates = Vec::with_capacity(DOWNWARD_KEYFRAME_SEARCH_LIMIT); - match (down_match, up_match) { - (Some(down), Some(up)) => { - if up.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) - <= down.mean_abs_diff_x100 + self.collect_fallback_downward_viewport_candidates(&frame, &mut candidates); + + match select_downward_viewport_candidate(&mut candidates) { + DownwardViewportResolution::NoMatch => { + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.fallback_downward_no_match", + ScrollDirection::Down, + None, + None, + Some(0), + Some("no_committed_keyframe_match"), + ); + + Ok(preview_update_outcome(preview_changed)) + }, + DownwardViewportResolution::Selected(candidate) => { + if self.fallback_downward_growth_exceeds_continuity_budget(candidate.viewport_top_y) { - self.observe_upward_rewind_from_committed(up.motion_rows); + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); self.log_decision( - "scroll_capture.fallback_preferred_upward_match", + "scroll_capture.fallback_downward_growth_blocked", ScrollDirection::Down, Some(MotionObservation { - direction: ScrollDirection::Up, - motion_rows: up.motion_rows, + direction: ScrollDirection::Down, + motion_rows: candidate.motion_rows, }), - None, - None, - Some("last_committed_overlap_preferred_upward_match"), + Some(candidate.viewport_top_y), + Some( + self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y), + ), + Some("fallback_committed_candidate_exceeded_local_continuity_budget"), ); - return Ok(ScrollObserveOutcome::UnsupportedDirection { - direction: ScrollDirection::Up, - }); - } - - if let Some(outcome) = self - .fallback_downward_growth_blocked_while_resume_frontier_active( - down.motion_rows, - preview_changed, - "resume_frontier_active_blocks_last_committed_fallback_downward_match", - previous_sample_frame.clone(), - previous_sample_fingerprint.clone(), - ) { - return Ok(outcome); + return Ok(preview_update_outcome(preview_changed)); } - self.last_motion_rows_hint = Some(down.motion_rows); - - let candidate_viewport_top_y = self - .current_viewport_top_y - .saturating_add(i32::try_from(down.motion_rows).unwrap_or_default()); - - self.observe_downward_growth_to_viewport( - frame, - candidate_viewport_top_y, - preview_changed, - Some(MotionObservation { - direction: ScrollDirection::Down, - motion_rows: down.motion_rows, - }), - "fallback_downward_match_from_last_committed_frame", - ) - }, - (Some(down), None) => { if let Some(outcome) = self .fallback_downward_growth_blocked_while_resume_frontier_active( - down.motion_rows, + candidate.viewport_top_y, + candidate.motion_rows, preview_changed, - "resume_frontier_active_blocks_last_committed_fallback_downward_match", - previous_sample_frame, - previous_sample_fingerprint, + "resume_frontier_active_blocks_keyframe_fallback_downward_match", ) { return Ok(outcome); } - self.last_motion_rows_hint = Some(down.motion_rows); - - let candidate_viewport_top_y = self - .current_viewport_top_y - .saturating_add(i32::try_from(down.motion_rows).unwrap_or_default()); - self.observe_downward_growth_to_viewport( frame, - candidate_viewport_top_y, + candidate.viewport_top_y, preview_changed, Some(MotionObservation { direction: ScrollDirection::Down, - motion_rows: down.motion_rows, + motion_rows: candidate.motion_rows, }), - "fallback_downward_match_without_upward_candidate", + candidate.source.fallback_decision_source(), ) }, - (None, Some(up)) => { - self.observe_upward_rewind_from_committed(up.motion_rows); + DownwardViewportResolution::Ambiguous { preferred, competing } => { + self.refresh_preview_only_downward_local_sample( + &frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); self.log_decision( - "scroll_capture.fallback_upward_match_only", + "scroll_capture.fallback_ambiguous_downward_registration", ScrollDirection::Down, Some(MotionObservation { - direction: ScrollDirection::Up, - motion_rows: up.motion_rows, + direction: ScrollDirection::Down, + motion_rows: preferred.motion_rows, }), - None, - None, - Some("last_committed_overlap_only_matched_upward"), + Some(preferred.viewport_top_y), + Some(0), + Some(preferred.competing_block_reason(competing)), ); - Ok(ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up }) + Ok(preview_update_outcome(preview_changed)) }, - (None, None) => Ok(preview_update_outcome(preview_changed)), } } @@ -1946,7 +4992,12 @@ impl ScrollSession { frame: RgbaImage, growth_rows: u32, viewport_top_y: i32, + decision_source: &'static str, + detected_motion_rows: Option, + effective_motion_rows_hint: Option, + previous_motion_rows_hint: Option, ) -> Result { + let fingerprint = scroll_capture_fingerprint(&frame); let strip = crop_bottom_rows(&frame, growth_rows) .ok_or_else(|| eyre::eyre!("failed to extract growth strip"))?; let preview_strip = resize_strip_to_preview_width(&strip, self.preview_width_px); @@ -1959,13 +5010,77 @@ impl ScrollSession { self.current_viewport_top_y = viewport_top_y; self.observed_viewport_top_y = viewport_top_y; + self.record_last_sample(&frame, fingerprint); + self.record_last_downward_observed_sample(&frame, scroll_capture_fingerprint(&frame)); + if self.should_seed_preview_only_local_after_observed_burst_commit( + decision_source, + growth_rows, + previous_motion_rows_hint, + ) { + self.record_preview_only_downward_local_sample(&frame, viewport_top_y); + self.seeded_preview_only_local_after_observed_burst_commit = true; + } else if self.should_preserve_preview_only_local_after_preview_only_burst_commit( + decision_source, + growth_rows, + previous_motion_rows_hint, + ) { + self.record_preview_only_downward_local_sample(&frame, viewport_top_y); + self.seeded_preview_only_local_after_observed_burst_commit = false; + self.last_blocked_preview_only_local_candidate = None; + } else { + self.clear_preview_only_downward_local_sample(); + } + self.last_unconfirmed_upward_fingerprint = None; self.last_committed_frame = frame.clone(); self.resume_frontier_top_y = None; self.resume_frontier_requires_reacquire = false; - self.growth_history.push(GrowthCommit { frame, growth_rows, viewport_top_y }); + self.growth_history.push(GrowthCommit { + frame, + growth_rows, + viewport_top_y, + decision_source, + detected_motion_rows, + effective_motion_rows_hint, + }); + + Ok(ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows }) + } + + fn should_seed_preview_only_local_after_observed_burst_commit( + &self, + decision_source: &'static str, + growth_rows: u32, + previous_motion_rows_hint: Option, + ) -> bool { + decision_source == DownwardViewportCandidateSource::ObservedSample.decision_source() + && self.transient_burst_search_enabled + && previous_motion_rows_hint.is_some_and(|previous| { + previous >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS && growth_rows < previous + }) + } - Ok(ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows }) + fn should_preserve_preview_only_local_after_preview_only_burst_commit( + &self, + decision_source: &'static str, + growth_rows: u32, + previous_motion_rows_hint: Option, + ) -> bool { + decision_source == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() + && previous_motion_rows_hint.is_some_and(|previous| { + if self.transient_burst_search_enabled { + growth_rows >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && growth_rows + >= previous.saturating_sub(PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS) + && growth_rows + <= previous + .saturating_add(PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS) + } else { + previous <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && growth_rows > 1 && growth_rows <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS + && growth_rows <= previous + } + }) } fn rebuild_export_image(&self) -> Result { @@ -1999,6 +5114,364 @@ struct DirectionMatch { motion_rows: u32, } +#[cfg(target_os = "macos")] +fn classify_vision_downward_sample_motion_against( + previous: &RgbaImage, + next: &RgbaImage, +) -> Option { + let previous_cg = cg_image_from_rgba_image(previous).ok()?; + let next_cg = cg_image_from_rgba_image(next).ok()?; + let options = NSDictionary::::new(); + let request = unsafe { + VNTranslationalImageRegistrationRequest::initWithTargetedCGImage_options( + VNTranslationalImageRegistrationRequest::alloc(), + previous_cg.as_ref(), + options.as_ref(), + ) + }; + let request_array = NSArray::from_retained_slice(&[request + .clone() + .into_super() + .into_super() + .into_super() + .into_super()]); + let handler = unsafe { + VNImageRequestHandler::initWithCGImage_options( + VNImageRequestHandler::alloc(), + next_cg.as_ref(), + options.as_ref(), + ) + }; + + handler.performRequests_error(request_array.as_ref()).ok()?; + + let results = unsafe { request.results() }?; + if results.count() == 0 { + return None; + } + + let translation = unsafe { results.objectAtIndex(0).alignmentTransform() }; + let motion_rows = translation.ty.round(); + if !motion_rows.is_finite() || motion_rows <= 0.0 { + return None; + } + let motion_rows = motion_rows as u32; + let config = OverlapSearchConfig::default(); + let matched = evaluate_overlap_direction( + previous, + next, + ScrollDirection::Down, + motion_rows..=motion_rows, + config, + )?; + let max_overlap = previous.height().min(next.height()); + + downward_registration_has_meaningful_overlap(matched, max_overlap, config).then_some(matched) +} + +#[cfg(not(target_os = "macos"))] +fn classify_vision_downward_sample_motion_against( + _previous: &RgbaImage, + _next: &RgbaImage, +) -> Option { + None +} + +fn estimate_pairwise_downward_shift_rows(previous: &RgbaImage, current: &RgbaImage) -> Option { + if previous.dimensions() != current.dimensions() { + return None; + } + let (_width, height) = previous.dimensions(); + if height < 3 { + return None; + } + let max_shift = height.saturating_sub(1); + + evaluate_overlap_direction( + previous, + current, + ScrollDirection::Down, + 1..=max_shift, + worker_pairwise_overlap_search_config(), + ) + .map(|matched| matched.motion_rows) +} + +#[cfg(target_os = "macos")] +fn cg_image_from_rgba_image( + image: &RgbaImage, +) -> Result> { + let width = image.width() as usize; + let height = image.height() as usize; + if width == 0 || height == 0 { + return Err(eyre::eyre!("vision registration image has zero dimensions")); + } + + let bytes = CFData::from_bytes(image.as_raw()); + let provider = CGDataProvider::with_cf_data(Some(bytes.as_ref())) + .ok_or_else(|| eyre::eyre!("failed to create CGDataProvider for Vision registration"))?; + let color_space = CGColorSpace::new_device_rgb() + .ok_or_else(|| eyre::eyre!("failed to create RGB colorspace for Vision registration"))?; + let bitmap_info = CGBitmapInfo(CGImageAlphaInfo::Last.0 | CGImageByteOrderInfo::Order32Big.0); + + unsafe { + CGImage::new( + width, + height, + 8, + 32, + width.saturating_mul(4), + Some(color_space.as_ref()), + bitmap_info, + Some(provider.as_ref()), + std::ptr::null(), + false, + CGColorRenderingIntent::RenderingIntentDefault, + ) + } + .ok_or_else(|| eyre::eyre!("failed to create CGImage for Vision registration")) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardRegistration { + NoMatch, + Matched(DirectionMatch), + Ambiguous { best: DirectionMatch, competing: DirectionMatch }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardSampleMatchSource { + ObservedSample, + PreviewOnlyLocalSample, +} + +impl DownwardSampleMatchSource { + const fn label(self) -> &'static str { + match self { + Self::ObservedSample => "observed_sample", + Self::PreviewOnlyLocalSample => "preview_only_local_sample", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DownwardSampleMatch { + matched: DirectionMatch, + source: DownwardSampleMatchSource, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardRegistrationWithSource { + NoMatch, + Matched(DownwardSampleMatch), + Ambiguous { best: DownwardSampleMatch, competing: DownwardSampleMatch }, +} + +impl DownwardRegistration { + fn map_source(self, source: DownwardSampleMatchSource) -> DownwardRegistrationWithSource { + match self { + Self::NoMatch => DownwardRegistrationWithSource::NoMatch, + Self::Matched(matched) => { + DownwardRegistrationWithSource::Matched(DownwardSampleMatch { matched, source }) + }, + Self::Ambiguous { best, competing } => DownwardRegistrationWithSource::Ambiguous { + best: DownwardSampleMatch { matched: best, source }, + competing: DownwardSampleMatch { matched: competing, source }, + }, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardViewportCandidateSource { + ObservedSample, + PreviewOnlyLocalSample, + CommittedKeyframe, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CommittedDownwardViewportCandidateMode { + LastCommittedOnly, + IncludeRecentHistory, +} + +impl DownwardViewportCandidateSource { + const fn priority(self) -> u8 { + match self { + Self::CommittedKeyframe => 0, + Self::ObservedSample => 1, + Self::PreviewOnlyLocalSample => 2, + } + } + + const fn decision_source(self) -> &'static str { + match self { + Self::ObservedSample => "sample_motion_downward_growth_from_observed_keyframe", + Self::PreviewOnlyLocalSample => { + "sample_motion_downward_growth_from_preview_only_local_sample" + }, + Self::CommittedKeyframe => "sample_motion_downward_growth_from_committed_keyframe", + } + } + + const fn fallback_decision_source(self) -> &'static str { + match self { + Self::ObservedSample => "fallback_downward_registration_from_observed_keyframe", + Self::PreviewOnlyLocalSample => { + "fallback_downward_registration_from_preview_only_local_sample" + }, + Self::CommittedKeyframe => "fallback_downward_registration_from_committed_keyframe", + } + } +} + +impl From for DownwardViewportCandidateSource { + fn from(value: DownwardSampleMatchSource) -> Self { + match value { + DownwardSampleMatchSource::ObservedSample => Self::ObservedSample, + DownwardSampleMatchSource::PreviewOnlyLocalSample => Self::PreviewOnlyLocalSample, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DownwardViewportCandidate { + source: DownwardViewportCandidateSource, + viewport_top_y: i32, + motion_rows: u32, + mean_abs_diff_x100: u32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct BlockedPreviewOnlyLocalCandidate { + candidate: DownwardViewportCandidate, + repeats: u8, +} + +impl DownwardViewportCandidate { + fn competing_block_reason(self, competing: Self) -> &'static str { + match (self.source, competing.source) { + ( + DownwardViewportCandidateSource::CommittedKeyframe, + DownwardViewportCandidateSource::CommittedKeyframe, + ) => "conflicting_committed_keyframe_authority", + _ => "conflicting_downward_viewport_authority", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardViewportResolution { + NoMatch, + Selected(DownwardViewportCandidate), + Ambiguous { preferred: DownwardViewportCandidate, competing: DownwardViewportCandidate }, +} + +fn select_downward_viewport_candidate( + candidates: &mut [DownwardViewportCandidate], +) -> DownwardViewportResolution { + if candidates.is_empty() { + return DownwardViewportResolution::NoMatch; + } + + if let Some(preferred_local) = prefer_local_downward_viewport_candidate(candidates) { + let competing = candidates.iter().copied().find(|candidate| { + candidate != &preferred_local + && candidate.viewport_top_y.abs_diff(preferred_local.viewport_top_y) + >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && candidate.mean_abs_diff_x100 + <= preferred_local + .mean_abs_diff_x100 + .saturating_add(DIRECTION_WARNING_MARGIN_X100) + }); + + return match competing { + Some(competing) => { + DownwardViewportResolution::Ambiguous { preferred: preferred_local, competing } + }, + None => DownwardViewportResolution::Selected(preferred_local), + }; + } + + candidates.sort_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.source.priority().cmp(&right.source.priority())) + .then(left.motion_rows.cmp(&right.motion_rows)) + }); + + let preferred = candidates[0]; + let competing = candidates.iter().copied().skip(1).find(|candidate| { + candidate.viewport_top_y.abs_diff(preferred.viewport_top_y) + >= DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS + && candidate.mean_abs_diff_x100 + <= preferred.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + }); + + match competing { + Some(competing) => DownwardViewportResolution::Ambiguous { preferred, competing }, + None => DownwardViewportResolution::Selected(preferred), + } +} + +fn format_downward_viewport_candidates(candidates: &[DownwardViewportCandidate]) -> String { + candidates + .iter() + .map(|candidate| { + format!( + "{:?}@{}/{}:{}", + candidate.source, + candidate.viewport_top_y, + candidate.motion_rows, + candidate.mean_abs_diff_x100 + ) + }) + .collect::>() + .join(",") +} + +fn prefer_local_downward_viewport_candidate( + candidates: &[DownwardViewportCandidate], +) -> Option { + let local = best_local_downward_viewport_candidate(candidates)?; + let committed = candidates + .iter() + .copied() + .filter(|candidate| candidate.source == DownwardViewportCandidateSource::CommittedKeyframe) + .min_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.motion_rows.cmp(&right.motion_rows)) + }); + + let Some(committed) = committed else { + return Some(local); + }; + + let committed_is_nearby = committed.viewport_top_y.abs_diff(local.viewport_top_y) + < DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS; + let committed_is_only_modestly_better = + committed.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + >= local.mean_abs_diff_x100; + + if committed_is_nearby && committed_is_only_modestly_better { Some(local) } else { None } +} + +fn best_local_downward_viewport_candidate( + candidates: &[DownwardViewportCandidate], +) -> Option { + candidates + .iter() + .copied() + .filter(|candidate| candidate.source != DownwardViewportCandidateSource::CommittedKeyframe) + .min_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.source.priority().cmp(&right.source.priority())) + .then(left.motion_rows.cmp(&right.motion_rows)) + }) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct OverlapSearchRange { start: u32, @@ -2086,6 +5559,9 @@ struct GrowthCommit { frame: RgbaImage, growth_rows: u32, viewport_top_y: i32, + decision_source: &'static str, + detected_motion_rows: Option, + effective_motion_rows_hint: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -2164,27 +5640,149 @@ fn evaluate_overlap_direction( range: RangeInclusive, config: OverlapSearchConfig, ) -> Option { + collect_overlap_direction_matches(previous, next, direction, range, config).into_iter().next() +} + +fn collect_overlap_direction_matches( + previous: &RgbaImage, + next: &RgbaImage, + direction: ScrollDirection, + range: RangeInclusive, + config: OverlapSearchConfig, +) -> Vec { + let Some(informative_span) = overlap_global_informative_span(previous, next) else { + return Vec::new(); + }; + let max_overlap = previous.height().min(next.height()); - let overlap = detect_vertical_overlap_in_range( - previous, - next, - range, - direction, - config, - overlap_global_informative_span(previous, next), - ); + let effective_min_overlap = + if max_overlap <= config.min_overlap_rows { 1 } else { config.min_overlap_rows.max(1) }; + let max_motion_rows = max_overlap.saturating_sub(effective_min_overlap).max(1); + let search_start = (*range.start()).max(1).min(max_motion_rows); + let search_end = (*range.end()).max(search_start).min(max_motion_rows); + let orientation = match direction { + ScrollDirection::Down => OverlapOrientation::PreviousBottomToNextTop, + ScrollDirection::Up => OverlapOrientation::PreviousTopToNextBottom, + }; + let mut matches = Vec::with_capacity(search_end.saturating_sub(search_start) as usize + 1); - if !overlap.matched { - return None; + for motion_rows in search_start..=search_end { + let overlap_rows = max_overlap.saturating_sub(motion_rows); + + if overlap_rows < effective_min_overlap { + continue; + } + + let diff = motion_mean_abs_diff_x100( + previous, + next, + motion_rows, + config, + orientation, + informative_span, + ); + + if diff > config.max_mean_abs_diff_x100 { + continue; + } + + matches.push(DirectionMatch { mean_abs_diff_x100: diff, motion_rows }); } - let motion_rows = max_overlap.saturating_sub(overlap.rows); + matches.sort_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.motion_rows.cmp(&right.motion_rows)) + }); + matches +} - if motion_rows == 0 { - return None; +fn collect_overlap_direction_matches_in_ranges( + previous: &RgbaImage, + next: &RgbaImage, + direction: ScrollDirection, + ranges: &[RangeInclusive], + config: OverlapSearchConfig, +) -> Vec { + let mut matches = Vec::new(); + + for range in ranges { + matches.extend(collect_overlap_direction_matches( + previous, + next, + direction, + range.clone(), + config, + )); + } + + if matches.len() <= 1 { + return matches; + } + + matches.sort_by(|left, right| { + left.motion_rows + .cmp(&right.motion_rows) + .then(left.mean_abs_diff_x100.cmp(&right.mean_abs_diff_x100)) + }); + + let mut deduped: Vec = Vec::with_capacity(matches.len()); + + for matched in matches { + if let Some(previous) = deduped.last_mut() + && previous.motion_rows == matched.motion_rows + { + if matched.mean_abs_diff_x100 < previous.mean_abs_diff_x100 { + *previous = matched; + } + continue; + } + + deduped.push(matched); + } + + deduped.sort_by(|left, right| { + left.mean_abs_diff_x100 + .cmp(&right.mean_abs_diff_x100) + .then(left.motion_rows.cmp(&right.motion_rows)) + }); + + deduped +} + +fn classify_downward_registration_candidates( + candidates: &[DirectionMatch], +) -> DownwardRegistration { + let Some(best) = candidates.first().copied() else { + return DownwardRegistration::NoMatch; + }; + let competing = candidates.iter().copied().skip(1).find(|candidate| { + candidate.motion_rows.abs_diff(best.motion_rows) >= DOWNWARD_REGISTRATION_AMBIGUOUS_GAP_ROWS + }); + + match competing { + Some(competing) + if best.mean_abs_diff_x100.saturating_add(DIRECTION_WARNING_MARGIN_X100) + >= competing.mean_abs_diff_x100 => + { + DownwardRegistration::Ambiguous { best, competing } + }, + _ => DownwardRegistration::Matched(best), } +} - Some(DirectionMatch { mean_abs_diff_x100: overlap.mean_abs_diff_x100, motion_rows }) +fn downward_registration_has_meaningful_overlap( + matched: DirectionMatch, + max_overlap: u32, + config: OverlapSearchConfig, +) -> bool { + let overlap_rows = max_overlap.saturating_sub(matched.motion_rows); + let effective_min_overlap = + if max_overlap <= config.min_overlap_rows { 1 } else { config.min_overlap_rows.max(1) }; + let min_overlap_rows = + effective_min_overlap.max(max_overlap / DOWNWARD_REGISTRATION_MIN_OVERLAP_DIVISOR).max(1); + + overlap_rows >= min_overlap_rows } fn preview_update_outcome(preview_changed: bool) -> ScrollObserveOutcome { @@ -2308,6 +5906,7 @@ fn max_directional_motion_rows( max_overlap.saturating_sub(effective_min_overlap).max(1) } +#[cfg(test)] fn detect_vertical_overlap_in_range( previous: &RgbaImage, next: &RgbaImage, @@ -2342,12 +5941,10 @@ fn detect_vertical_overlap_in_range( continue; } - let band_rows = overlap_rows.clamp(1, MOTION_SEARCH_BAND_ROWS); let diff = motion_mean_abs_diff_x100( previous, next, motion_rows, - band_rows, config, orientation, informative_span, @@ -2379,6 +5976,31 @@ fn resize_strip_to_preview_width(strip: &RgbaImage, preview_width_px: u32) -> Rg imageops::resize(strip, preview_width_px, preview_height, FilterType::Triangle) } +pub(crate) fn compose_provisional_preview_image( + base_preview: &RgbaImage, + latest_frame: Option<&RgbaImage>, + motion_rows_hint: Option, + preview_width_px: u32, +) -> RgbaImage { + let Some(frame) = latest_frame else { + return base_preview.clone(); + }; + let Some(motion_rows_hint) = motion_rows_hint else { + return base_preview.clone(); + }; + let hinted_growth_rows = motion_rows_hint.min(frame.height()); + if hinted_growth_rows == 0 { + return base_preview.clone(); + } + + let Some(strip) = crop_bottom_rows(frame, hinted_growth_rows) else { + return base_preview.clone(); + }; + let preview_strip = resize_strip_to_preview_width(&strip, preview_width_px); + + append_vertical_image(base_preview, &preview_strip).unwrap_or_else(|_| base_preview.clone()) +} + fn crop_bottom_rows(frame: &RgbaImage, rows: u32) -> Option { let rows = rows.min(frame.height()); @@ -2437,7 +6059,6 @@ fn motion_mean_abs_diff_x100( previous: &RgbaImage, next: &RgbaImage, motion_rows: u32, - band_rows: u32, config: OverlapSearchConfig, orientation: OverlapOrientation, informative_span: InformativeSpan, @@ -2450,9 +6071,8 @@ fn motion_mean_abs_diff_x100( return u32::MAX; } - let band_rows = band_rows.min(overlap_rows).max(1); let column_samples = width.min(config.max_column_samples).max(1); - let row_samples = band_rows.min(config.max_row_samples).max(1); + let row_samples = overlap_rows.min(config.max_row_samples).max(1); let previous_overlap_start_y = previous.height().saturating_sub(overlap_rows); let next_overlap_start_y = next.height().saturating_sub(overlap_rows); let previous_start_y = match orientation { @@ -2471,7 +6091,7 @@ fn motion_mean_abs_diff_x100( let mut comparisons = 0_u64; for row in 0..row_samples { - let local_y = evenly_spaced_sample(0, band_rows, row, row_samples); + let local_y = evenly_spaced_sample(0, overlap_rows, row, row_samples); let previous_y = previous_start_y.saturating_add(local_y).min(previous.height().saturating_sub(1)); let next_y = next_start_y.saturating_add(local_y).min(next.height().saturating_sub(1)); @@ -2503,9 +6123,9 @@ fn overlap_global_informative_span(left: &RgbaImage, right: &RgbaImage) -> Optio match (left_span, right_span) { (Some(left_span), Some(right_span)) => { - let start_x = left_span.start_x.min(right_span.start_x); + let start_x = left_span.start_x.max(right_span.start_x); let end_exclusive_x = - left_span.end_exclusive_x.max(right_span.end_exclusive_x).min(width); + left_span.end_exclusive_x.min(right_span.end_exclusive_x).min(width); (end_exclusive_x > start_x).then_some(InformativeSpan { start_x, end_exclusive_x }) }, @@ -2592,8 +6212,12 @@ mod tests { use image::Rgba; use crate::scroll_capture::{ - self, DirectionMatch, MotionObservation, OverlapSearchConfig, ScrollDirection, - ScrollFrameFingerprint, ScrollObserveOutcome, ScrollSession, + self, DirectionMatch, DownwardRegistration, DownwardSampleMatch, DownwardSampleMatchSource, + DownwardViewportCandidate, DownwardViewportCandidateSource, DownwardViewportResolution, + GrowthCommit, MotionObservation, OverlapSearchConfig, PreviewOnlyDownwardLocalSample, + ScrollDirection, ScrollFrameFingerprint, ScrollObserveOutcome, ScrollSession, + classify_vision_downward_sample_motion_against, estimate_pairwise_downward_shift_rows, + select_downward_viewport_candidate, }; fn make_test_image(width: u32, rows: &[[u8; 4]]) -> image::RgbaImage { @@ -2617,27 +6241,99 @@ mod tests { make_test_image(width, &document[start_row..start_row + window_rows]) } - fn paint_row(frame: &mut image::RgbaImage, row: u32, color: [u8; 4]) { - for x in 0..frame.width() { - frame.put_pixel(x, row, Rgba(color)); - } - } - fn make_sparse_textlike_window(width: u32, height: u32, start_row: u32) -> image::RgbaImage { let stripe_x = 104_u32; let mut image = image::RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255])); for y in 0..height { let document_row = start_row.saturating_add(y); - let shade = ((document_row.saturating_mul(17)) % 180) as u8; + let shade = ((document_row.saturating_mul(17)) % 180) as u8; + + for x in stripe_x..stripe_x.saturating_add(6) { + image.put_pixel(x, y, Rgba([shade, shade, shade, 255])); + } + for x in stripe_x.saturating_add(10)..stripe_x.saturating_add(13) { + if document_row % 19 < 9 { + image.put_pixel(x, y, Rgba([40, 40, 40, 255])); + } + } + } + + image + } + + fn make_sparse_textlike_window_with_moving_edge_scrollbar( + width: u32, + height: u32, + start_row: u32, + thumb_top: u32, + ) -> image::RgbaImage { + let mut image = make_sparse_textlike_window(width, height, start_row); + let track_left = width.saturating_sub(18); + let thumb_height = (height / 4).max(12).min(height.max(1)); + let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); + let thumb_right = width.saturating_sub(3).max(track_left.saturating_add(4)); + + for y in 0..height { + for x in track_left..width { + image.put_pixel(x, y, Rgba([224, 224, 224, 255])); + } + } + + for y in thumb_top..thumb_top.saturating_add(thumb_height) { + for x in track_left.saturating_add(3)..thumb_right { + image.put_pixel(x, y, Rgba([28, 28, 28, 255])); + } + } + + image + } + + fn make_browser_like_window(width: u32, height: u32, start_row: u32) -> image::RgbaImage { + let mut image = make_sparse_textlike_window(width, height, start_row); + let scrollbar_left = width.saturating_sub(18); + let content_left = 56_u32; + let content_right = width.saturating_sub(48); + let heading_width = 220_u32; + let paragraph_width = content_right.saturating_sub(content_left); + + for y in 0..height { + let document_row = start_row.saturating_add(y); - for x in stripe_x..stripe_x.saturating_add(6) { - image.put_pixel(x, y, Rgba([shade, shade, shade, 255])); - } - for x in stripe_x.saturating_add(10)..stripe_x.saturating_add(13) { - if document_row % 19 < 9 { - image.put_pixel(x, y, Rgba([40, 40, 40, 255])); + if document_row % 420 < 18 { + for x in content_left..content_left.saturating_add(heading_width) { + image.put_pixel(x, y, Rgba([26, 26, 26, 255])); } + } else if document_row % 420 >= 54 && document_row % 420 < 220 { + if document_row % 24 < 3 { + let trim = ((document_row / 24) % 5) * 18; + for x in content_left + ..content_left.saturating_add(paragraph_width.saturating_sub(trim)) + { + image.put_pixel(x, y, Rgba([72, 72, 72, 255])); + } + } + } else if document_row % 420 >= 270 && document_row % 420 < 360 { + if document_row % 20 < 2 { + for x in content_left.saturating_add(20) + ..content_left.saturating_add(paragraph_width.saturating_sub(70)) + { + image.put_pixel(x, y, Rgba([98, 98, 98, 255])); + } + } + } + + for x in scrollbar_left..width { + image.put_pixel(x, y, Rgba([232, 232, 232, 255])); + } + } + + let thumb_height = (height / 5).max(16); + let thumb_top = (start_row / 3) % height.max(thumb_height + 1); + let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); + for y in thumb_top..thumb_top.saturating_add(thumb_height) { + for x in scrollbar_left.saturating_add(3)..width.saturating_sub(4) { + image.put_pixel(x, y, Rgba([96, 96, 96, 255])); } } @@ -2711,6 +6407,379 @@ mod tests { assert_eq!(session.export_image().get_pixel(0, 5), &Rgba([60, 0, 0, 255])); } + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_commits_substantial_downward_growth_with_corroboration() { + let base = make_sparse_textlike_window(512, 640, 0); + let moved = make_sparse_textlike_window(512, 640, 90); + let matched = classify_vision_downward_sample_motion_against(&base, &moved) + .expect("vision registration should detect the substantial downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + let outcome = session.observe_worker_pairwise_vision_frame(moved).unwrap(); + + assert!(matched.motion_rows >= 32); + assert_eq!( + outcome, + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + assert_eq!(session.export_image().height(), 640 + matched.motion_rows); + assert_eq!(session.current_viewport_top_y(), i32::try_from(matched.motion_rows).unwrap()); + } + + #[test] + fn pairwise_downward_shift_estimate_matches_sparse_textlike_motion() { + let base = make_sparse_textlike_window(512, 640, 0); + let moved = make_sparse_textlike_window(512, 640, 58); + + assert_eq!(estimate_pairwise_downward_shift_rows(&base, &moved), Some(58)); + } + + #[test] + fn pairwise_downward_shift_estimate_matches_browser_like_motion_above_legacy_cap() { + let base = make_browser_like_window(512, 640, 0); + let moved = make_browser_like_window(512, 640, 320); + + assert_eq!(estimate_pairwise_downward_shift_rows(&base, &moved), Some(320)); + } + + #[test] + fn pairwise_downward_shift_estimate_tracks_successive_browser_like_steps() { + let frames = [0_u32, 180, 360, 540, 720] + .into_iter() + .map(|start_row| make_browser_like_window(512, 640, start_row)) + .collect::>(); + + for window in frames.windows(2) { + assert_eq!(estimate_pairwise_downward_shift_rows(&window[0], &window[1]), Some(180)); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_uses_latest_committed_live_frame_for_followup_growth() { + let base = make_sparse_textlike_window(512, 640, 0); + let step_one = make_sparse_textlike_window(512, 640, 180); + let step_two = make_sparse_textlike_window(512, 640, 360); + let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first pairwise registration should detect downward motion"); + let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup pairwise registration should detect downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_one).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: first_match.motion_rows, + } + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_two).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: followup_match.motion_rows, + } + ); + assert_eq!( + session.export_image().height(), + 640 + first_match.motion_rows + followup_match.motion_rows + ); + assert_eq!( + session.current_viewport_top_y(), + i32::try_from(first_match.motion_rows + followup_match.motion_rows).unwrap() + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_handles_repeated_frame_between_growth_steps() { + let base = make_sparse_textlike_window(512, 640, 0); + let step_one = make_sparse_textlike_window(512, 640, 180); + let step_two = make_sparse_textlike_window(512, 640, 360); + let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first pairwise registration should detect downward motion"); + let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup pairwise registration should detect downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_one.clone()).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: first_match.motion_rows, + } + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_one).unwrap(), + ScrollObserveOutcome::NoChange + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_two).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: followup_match.motion_rows, + } + ); + assert_eq!( + session.export_image().height(), + 640 + first_match.motion_rows + followup_match.motion_rows + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_recovers_after_blocked_overshot_frame() { + let base = make_browser_like_window(512, 640, 0); + let blocked = make_browser_like_window(512, 640, 760); + let followup = make_browser_like_window(512, 640, 844); + let matched = classify_vision_downward_sample_motion_against(&blocked, &followup).expect( + "pairwise registration should detect the followup step after the blocked overshot", + ); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(blocked).unwrap(), + ScrollObserveOutcome::NoChange + ); + assert_eq!(session.export_image().height(), 640); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!( + session.observe_worker_pairwise_vision_frame(followup).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + assert_eq!(session.export_image().height(), 640 + matched.motion_rows); + assert_eq!(session.current_viewport_top_y(), i32::try_from(matched.motion_rows).unwrap()); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_clears_preview_local_followup_carryover_on_no_change() { + let base = make_sparse_textlike_window(512, 640, 0); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + session.record_preview_only_downward_local_sample(&base, 123); + session.pending_suppressed_huge_preview_only_local_followup = + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 160, + motion_rows: 160, + mean_abs_diff_x100: 0, + }); + session.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 2; + session.pending_extreme_preview_only_local_tail_followup = + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 161, + motion_rows: 1, + mean_abs_diff_x100: 0, + }); + session.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 1; + + assert_eq!( + session.observe_worker_pairwise_vision_frame(base).unwrap(), + ScrollObserveOutcome::NoChange + ); + assert!(session.last_preview_only_downward_local_sample.is_none()); + assert!(session.pending_suppressed_huge_preview_only_local_followup.is_none()); + assert_eq!(session.pending_suppressed_huge_preview_only_local_followup_remaining_blocks, 0); + assert!(session.pending_extreme_preview_only_local_tail_followup.is_none()); + assert_eq!(session.pending_extreme_preview_only_local_tail_followup_remaining_blocks, 0); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_clears_preview_local_followup_carryover_on_commit() { + let base = make_sparse_textlike_window(512, 640, 0); + let moved = make_sparse_textlike_window(512, 640, 180); + let matched = classify_vision_downward_sample_motion_against(&base, &moved) + .expect("pairwise registration should detect downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + session.record_preview_only_downward_local_sample(&moved, 180); + session.pending_suppressed_huge_preview_only_local_followup = + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 160, + motion_rows: 160, + mean_abs_diff_x100: 0, + }); + session.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 2; + session.pending_extreme_preview_only_local_tail_followup = + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 161, + motion_rows: 1, + mean_abs_diff_x100: 0, + }); + session.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 1; + + assert_eq!( + session.observe_worker_pairwise_vision_frame(moved).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + assert!(session.last_preview_only_downward_local_sample.is_none()); + assert!(session.pending_suppressed_huge_preview_only_local_followup.is_none()); + assert_eq!(session.pending_suppressed_huge_preview_only_local_followup_remaining_blocks, 0); + assert!(session.pending_extreme_preview_only_local_tail_followup.is_none()); + assert_eq!(session.pending_extreme_preview_only_local_tail_followup_remaining_blocks, 0); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_commits_successive_slowdown_steps() { + let frames = [0_u32, 180, 300, 380, 420] + .into_iter() + .map(|start_row| make_sparse_textlike_window(512, 640, start_row)) + .collect::>(); + let mut session = ScrollSession::new(frames[0].clone(), 320).unwrap(); + let mut expected_export_height = 640_u32; + let mut expected_viewport_top_y = 0_i32; + + for window in frames.windows(2) { + let previous = &window[0]; + let next = window[1].clone(); + let matched = classify_vision_downward_sample_motion_against(previous, &next) + .expect("pairwise registration should detect each slowdown step"); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(next).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + + expected_export_height = expected_export_height.saturating_add(matched.motion_rows); + expected_viewport_top_y += i32::try_from(matched.motion_rows).unwrap(); + } + + assert_eq!(session.export_image().height(), expected_export_height); + assert_eq!(session.current_viewport_top_y(), expected_viewport_top_y); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_commits_browser_like_growth_above_legacy_cap() { + let base = make_browser_like_window(512, 640, 0); + let moved = make_browser_like_window(512, 640, 320); + let matched = classify_vision_downward_sample_motion_against(&base, &moved) + .expect("vision registration should detect the browser-like downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert!(matched.motion_rows > 256); + assert_eq!( + session.observe_worker_pairwise_vision_frame(moved).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + assert_eq!(session.export_image().height(), 640 + matched.motion_rows); + assert_eq!(session.current_viewport_top_y(), i32::try_from(matched.motion_rows).unwrap()); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_commits_successive_browser_like_steps() { + let frames = [0_u32, 180, 360, 540, 720] + .into_iter() + .map(|start_row| make_browser_like_window(512, 640, start_row)) + .collect::>(); + let mut session = ScrollSession::new(frames[0].clone(), 320).unwrap(); + let mut expected_export_height = 640_u32; + let mut expected_viewport_top_y = 0_i32; + + for window in frames.windows(2) { + let previous = &window[0]; + let next = window[1].clone(); + let matched = classify_vision_downward_sample_motion_against(previous, &next) + .expect("pairwise registration should detect each browser-like step"); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(next).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + + expected_export_height = expected_export_height.saturating_add(matched.motion_rows); + expected_viewport_top_y += i32::try_from(matched.motion_rows).unwrap(); + } + + assert_eq!(session.export_image().height(), expected_export_height); + assert_eq!(session.current_viewport_top_y(), expected_viewport_top_y); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_handles_repeated_browser_like_frame_between_growth_steps() { + let base = make_browser_like_window(512, 640, 0); + let step_one = make_browser_like_window(512, 640, 180); + let step_two = make_browser_like_window(512, 640, 360); + let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first browser-like step should register downward motion"); + let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup browser-like step should register downward motion"); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_one.clone()).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: first_match.motion_rows, + } + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_one).unwrap(), + ScrollObserveOutcome::NoChange + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(step_two).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: followup_match.motion_rows, + } + ); + assert_eq!( + session.export_image().height(), + 640 + first_match.motion_rows + followup_match.motion_rows + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn worker_pairwise_vision_browser_like_followup_uses_adjacent_worker_frame() { + let base = make_browser_like_window(512, 640, 0); + let blocked = make_browser_like_window(512, 640, 700); + let followup = make_browser_like_window(512, 640, 784); + let matched = classify_vision_downward_sample_motion_against(&blocked, &followup).expect( + "browser-like pairwise registration should use the immediately previous worker frame", + ); + let mut session = ScrollSession::new(base, 320).unwrap(); + + assert_eq!( + session.observe_worker_pairwise_vision_frame(blocked).unwrap(), + ScrollObserveOutcome::NoChange + ); + assert_eq!( + session.observe_worker_pairwise_vision_frame(followup).unwrap(), + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: matched.motion_rows, + } + ); + assert_eq!(session.export_image().height(), 640 + matched.motion_rows); + assert_eq!(session.current_viewport_top_y(), i32::try_from(matched.motion_rows).unwrap()); + } + #[test] fn session_supports_multiple_downward_growth_steps() { let document = [ @@ -2782,6 +6851,7 @@ mod tests { outcome, ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange )); assert_eq!(session.export_image(), &base); } @@ -2808,6 +6878,7 @@ mod tests { session.observe_downward_sample(make_window(&document, 3, start_row, 5)).unwrap(), ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange )); assert_eq!(session.export_image().height(), initial_height); } @@ -2849,6 +6920,46 @@ mod tests { assert_eq!(session.export_image().height(), 129); } + #[test] + fn session_commits_growth_with_sparse_columns_and_moving_edge_scrollbar() { + let base = make_sparse_textlike_window_with_moving_edge_scrollbar(256, 120, 0, 8); + let moved = make_sparse_textlike_window_with_moving_edge_scrollbar(256, 120, 9, 40); + let mut session = ScrollSession::new(base, 320).unwrap(); + let outcome = session.observe_downward_sample(moved).unwrap(); + + assert_eq!( + outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 9 } + ); + assert_eq!(session.export_image().height(), 129); + } + + #[test] + fn repeated_periodic_content_fails_closed_when_downward_registration_is_ambiguous() { + let document: Vec<[u8; 4]> = (0..256) + .map(|row| { + let bucket = (row % 32) as u8; + + [ + bucket.saturating_mul(7), + 255_u8.saturating_sub(bucket.saturating_mul(3)), + bucket.saturating_mul(5), + 255, + ] + }) + .collect(); + let base = make_window(&document, 8, 0, 96); + let moved = make_window(&document, 8, 24, 96); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + assert!(matches!( + session.observe_downward_sample(moved).unwrap(), + ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.export_image(), &base); + assert_eq!(session.current_viewport_top_y, 0); + } + #[test] fn sparse_textlike_small_downward_steps_eventually_append() { let base = make_sparse_textlike_window(256, 120, 0); @@ -2871,6 +6982,31 @@ mod tests { assert!(session.export_image().height() > initial_height); } + #[test] + fn observed_sample_requires_meaningful_overlap_before_committing_large_motion() { + let document = (0_u16..320) + .map(|row| { + [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] + }) + .collect::>(); + let base = make_window(&document, 3, 0, 160); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + session.last_motion_rows_hint = Some(128); + + let far = make_window(&document, 3, 130, 160); + let export_before = session.export_image().clone(); + let preview_before = session.preview_image().clone(); + + assert!(matches!( + session.observe_downward_sample(far).unwrap(), + ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange + )); + assert_eq!(session.export_image(), &export_before); + assert_eq!(session.preview_image(), &preview_before); + assert_eq!(session.current_viewport_top_y, 0); + } + #[test] fn periodic_far_downward_frame_does_not_use_full_range_fallback_after_local_miss() { let document = (0_u16..128) @@ -2914,6 +7050,79 @@ mod tests { assert_eq!(session.preview_image(), &preview_before); } + #[test] + fn committed_growth_rewrites_motion_hint_to_actual_growth_rows() { + let document = (0_u16..160) + .map(|row| { + [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] + }) + .collect::>(); + let mut session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + + assert_eq!( + session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } + ); + assert_eq!(session.last_motion_rows_hint, Some(20)); + assert_eq!( + session + .observe_downward_growth_to_viewport( + make_window(&document, 3, 24, 48), + 24, + true, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows: 64 }), + "test_residual_growth_rewrites_hint", + ) + .unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 4 } + ); + assert_eq!(session.last_motion_rows_hint, Some(4)); + } + + #[test] + fn hinted_downward_registration_does_not_escape_to_far_full_range_match() { + let document = (0_u16..320) + .map(|row| { + [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] + }) + .collect::>(); + let previous = make_window(&document, 3, 0, 160); + let next = make_window(&document, 3, 100, 160); + let session = ScrollSession::new(previous.clone(), 320).unwrap(); + + assert!(matches!( + session.evaluate_reference_downward_registration(&previous, &next, None, true), + DownwardRegistration::Matched(DirectionMatch { motion_rows: 100, .. }) + )); + assert_eq!( + session.evaluate_reference_downward_registration(&previous, &next, Some(20), true), + DownwardRegistration::NoMatch + ); + } + + #[test] + fn active_preview_helpers_stay_committed_even_with_provisional_like_session_state() { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + ]; + let base = make_window(&document, 3, 0, 5); + let latest = make_window(&document, 3, 1, 5); + let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + + session.last_sample_frame = latest.clone(); + session.observed_viewport_top_y = 1; + + assert_eq!(session.preview_display_mode(), "committed"); + assert_eq!(session.preview_display_image(), session.export_image().clone()); + } + #[test] fn upward_motion_does_not_reset_downward_progress() { let document = [ @@ -2936,10 +7145,15 @@ mod tests { session.observe_downward_sample(make_window(&document, 3, 0, 5)).unwrap(), ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange )); + let resume_outcome = + session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(); assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + resume_outcome, + ScrollObserveOutcome::NoChange + | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } )); assert_eq!( session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), @@ -3017,18 +7231,25 @@ mod tests { session.observe_downward_sample(make_window(&document, 3, 0, 5)).unwrap(), ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange )); let height_after_upward_rewind = session.export_image().height(); assert!(matches!( session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + ScrollObserveOutcome::NoChange + | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } )); assert_eq!(session.export_image().height(), height_after_upward_rewind); + let partial_resume_outcome = + session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(); assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + partial_resume_outcome, + ScrollObserveOutcome::NoChange + | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } )); assert_eq!(session.export_image().height(), height_after_upward_rewind); assert_eq!( @@ -3066,10 +7287,15 @@ mod tests { session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::NoChange )); + let return_outcome = + session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(); assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + return_outcome, + ScrollObserveOutcome::NoChange + | ScrollObserveOutcome::PreviewUpdated + | ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } )); assert_eq!(session.export_image().height(), height_before_resume); assert_eq!( @@ -3082,7 +7308,7 @@ mod tests { } #[test] - fn resume_frontier_blocks_repeated_return_to_last_committed_viewport() { + fn downward_input_upward_like_frame_does_not_arm_resume_frontier_or_poison_sample() { let document = [ [10, 0, 0, 255], [20, 0, 0, 255], @@ -3104,1030 +7330,1856 @@ mod tests { ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } ); - let height_after_second_append = session.export_image().height(); + let sample_before = session.last_sample_frame.clone(); + let sample_fingerprint_before = session.last_sample_fingerprint.clone(); + let height_before = session.export_image().height(); assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.observed_viewport_top_y, 1); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.observed_viewport_top_y, 2); - assert_eq!(session.export_image().height(), height_after_second_append); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), + session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } | ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange )); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.observed_viewport_top_y, 1); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.resume_frontier_top_y, Some(2)); + assert_eq!(session.export_image().height(), height_before); + assert_eq!(session.current_viewport_top_y, 2); assert_eq!(session.observed_viewport_top_y, 2); - assert_eq!(session.export_image().height(), height_after_second_append); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 3, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); assert_eq!(session.resume_frontier_top_y, None); - assert_eq!(session.export_image().height(), 8); - assert_eq!(session.export_image().get_pixel(0, 0), &Rgba([10, 0, 0, 255])); - assert_eq!(session.export_image().get_pixel(0, 7), &Rgba([80, 0, 0, 255])); + assert!(!session.resume_frontier_requires_reacquire); + assert_eq!(session.last_sample_frame, sample_before); + assert_eq!(session.last_sample_fingerprint, sample_fingerprint_before); } #[test] - fn upward_input_uses_last_committed_match_when_sample_motion_looks_downward() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn viewport_selection_fails_closed_when_observed_and_committed_authority_conflict() { + let observed = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 120, + motion_rows: 20, + mean_abs_diff_x100: 100, + }; + let committed = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 360, + motion_rows: 260, + mean_abs_diff_x100: 90, + }; + let mut candidates = [observed, committed]; assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + select_downward_viewport_candidate(&mut candidates), + DownwardViewportResolution::Ambiguous { preferred: committed, competing: observed } ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + } + + #[test] + fn committed_keyframe_candidate_requires_meaningful_overlap() { + let document = (0_u16..96) + .map(|row| { + [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] + }) + .collect::>(); + let session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + let mut candidates = Vec::new(); + + session.push_downward_viewport_candidate( + &session.anchor_frame, + 0, + &make_window(&document, 3, 40, 48), + DownwardViewportCandidateSource::CommittedKeyframe, + &mut candidates, ); - let stale_lower_sample = make_window(&document, 3, 0, 5); + assert!(candidates.is_empty()); + } - session.last_sample_frame = stale_lower_sample.clone(); - session.last_sample_fingerprint = - Some(super::scroll_capture_fingerprint(&stale_lower_sample)); + #[test] + fn committed_fallback_can_recover_from_an_older_recent_keyframe() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); assert_eq!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } + session.observe_downward_sample(make_sparse_textlike_window(256, 120, 18)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 18 } + ); + assert_eq!( + session.observe_downward_sample(make_sparse_textlike_window(256, 120, 29)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 11 } ); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.observed_viewport_top_y, 1); - let height_before_return = session.export_image().height(); + session.last_committed_frame = + image::RgbaImage::from_pixel(256, 120, Rgba([255, 255, 255, 255])); + let target = make_sparse_textlike_window(256, 120, 39); + let mut candidates = Vec::new(); + + session.collect_committed_downward_viewport_candidates(&target, &mut candidates); + + assert!(candidates.iter().any(|candidate| { + candidate.source == DownwardViewportCandidateSource::CommittedKeyframe + && candidate.viewport_top_y == 39 + })); + } + + #[test] + fn fallback_committed_candidates_ignore_older_recent_keyframes() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_return); assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 3, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + session.observe_downward_sample(make_sparse_textlike_window(256, 120, 18)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 18 } ); - assert_eq!(session.export_image().height(), height_before_return + 1); + assert_eq!( + session.observe_downward_sample(make_sparse_textlike_window(256, 120, 29)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 11 } + ); + + session.last_committed_frame = + image::RgbaImage::from_pixel(256, 120, Rgba([255, 255, 255, 255])); + let target = make_sparse_textlike_window(256, 120, 39); + let mut candidates = Vec::new(); + + session.collect_fallback_downward_viewport_candidates(&target, &mut candidates); + + assert!(candidates.is_empty()); } #[test] - fn blocked_fallback_downward_input_does_not_append_or_advance_sample_baseline() { - let document = [ - [0, 0, 0, 255], - [200, 0, 0, 255], - [40, 0, 0, 255], - [240, 0, 0, 255], - [80, 0, 0, 255], - [180, 0, 0, 255], - [20, 0, 0, 255], - [220, 0, 0, 255], - [60, 0, 0, 255], - [160, 0, 0, 255], - [100, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn fallback_committed_growth_respects_local_continuity_budget() { + let document = (0_u16..220) + .map(|row| { + [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] + }) + .collect::>(); + let mut session = ScrollSession::new(make_window(&document, 3, 0, 64), 320).unwrap(); assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + session.observe_downward_sample(make_window(&document, 3, 20, 64)).unwrap(), + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } ); + + session.last_motion_rows_hint = Some(2); + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_window(&document, 3, 24, 64), + viewport_top_y: 24, + }); + + assert!(session.fallback_downward_growth_exceeds_continuity_budget(33)); + assert!(!session.fallback_downward_growth_exceeds_continuity_budget(32)); + } + + #[test] + fn nearby_local_candidate_wins_when_committed_is_only_modestly_better() { + let observed = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 132, + motion_rows: 12, + mean_abs_diff_x100: 120, + }; + let committed = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 130, + motion_rows: 10, + mean_abs_diff_x100: 80, + }; + let mut candidates = [observed, committed]; + assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + select_downward_viewport_candidate(&mut candidates), + DownwardViewportResolution::Selected(observed) ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - )); + } - let blocked_frame = make_window(&document, 3, 6, 5); - let unrelated_sample = make_test_image( - 3, - &[ - [255, 255, 0, 255], - [0, 255, 255, 255], - [255, 0, 255, 255], - [0, 0, 255, 255], - [255, 255, 255, 255], - ], - ); + #[test] + fn burst_observed_sample_candidate_is_suppressed_when_it_far_exceeds_local_continuity_budget() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 14; + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(1_219); + session.transient_burst_search_enabled = true; - session.last_sample_frame = unrelated_sample.clone(); - session.last_sample_fingerprint = - Some(super::scroll_capture_fingerprint(&unrelated_sample)); + assert!(session.should_suppress_observed_sample_candidate(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 419, + motion_rows: 413, + mean_abs_diff_x100: 0, + })); + } + + #[test] + fn burst_observed_sample_candidate_is_kept_when_it_stays_within_local_continuity_budget() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - let height_before_fallback = session.export_image().height(); - let observed_before_fallback = session.observed_viewport_top_y; - let sample_before_fallback = session.last_sample_frame.clone(); - let sample_fingerprint_before_fallback = session.last_sample_fingerprint.clone(); + session.current_viewport_top_y = 14; + session.last_motion_rows_hint = Some(9); + session.transient_motion_rows_hint = Some(74); + session.transient_burst_search_enabled = true; - assert!(matches!( - session.observe_downward_sample(blocked_frame).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + assert!(!session.should_suppress_observed_sample_candidate(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 30, + motion_rows: 16, + mean_abs_diff_x100: 0, + })); + } + + #[test] + fn burst_observed_sample_candidate_near_recent_continuity_can_exceed_budget_without_suppression() + { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 130; + session.last_motion_rows_hint = Some(38); + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + + assert!(!session.should_suppress_observed_sample_candidate(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 162, + motion_rows: 32, + mean_abs_diff_x100: 533, + })); + } + + #[test] + fn burst_observed_sample_candidate_near_recent_continuity_still_suppresses_when_diff_is_too_high() + { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 14; + session.last_motion_rows_hint = Some(9); + session.transient_motion_rows_hint = Some(1_219); + session.transient_burst_search_enabled = true; + + assert!(session.should_suppress_observed_sample_candidate(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 31, + motion_rows: 17, + mean_abs_diff_x100: 733, + })); + } + + #[test] + fn corroborated_observed_candidate_can_recover_after_initial_continuity_suppression() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 149; + session.last_motion_rows_hint = Some(16); + session.transient_motion_rows_hint = Some(12); + session.transient_burst_search_enabled = true; + + let candidate = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 169, + motion_rows: 20, + mean_abs_diff_x100: 0, + }; + let mut candidates = vec![DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 169, + motion_rows: 20, + mean_abs_diff_x100: 0, + }]; + + assert!(session.should_suppress_observed_sample_candidate(candidate)); + + session.restore_corroborated_observed_candidate(Some(candidate), &mut candidates); + + assert!(candidates.contains(&candidate)); + } + + #[test] + fn tiny_observed_recovery_fails_closed_during_large_transient_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 261; + session.last_motion_rows_hint = Some(24); + session.transient_motion_rows_hint = Some(86); + session.transient_burst_search_enabled = true; + + assert!(session.should_fail_closed_tiny_observed_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 263, + motion_rows: 2, + mean_abs_diff_x100: 0, + } )); - assert_eq!(session.export_image().height(), height_before_fallback); - assert_eq!(session.observed_viewport_top_y, observed_before_fallback); - assert_eq!(session.last_sample_frame, sample_before_fallback); - assert_eq!(session.last_sample_fingerprint, sample_fingerprint_before_fallback); - assert_eq!(session.export_image().get_pixel(0, 0), &Rgba([0, 0, 0, 255])); - assert_eq!(session.export_image().get_pixel(0, 6), &Rgba([20, 0, 0, 255])); } #[test] - fn resume_frontier_direct_gating_ignores_stale_observed_position() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - [90, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn tiny_observed_recovery_does_not_block_when_recent_continuity_is_small() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 14; + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(1_217); + session.transient_burst_search_enabled = true; + + assert!(!session.should_fail_closed_tiny_observed_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 15, + motion_rows: 1, + mean_abs_diff_x100: 0, + } + )); + } - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + #[test] + fn outsized_observed_recovery_after_one_pixel_preview_local_commit_fails_closed() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 74; + session.last_motion_rows_hint = Some(1); + session.transient_motion_rows_hint = Some(277); + session.transient_burst_search_enabled = true; + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 74), + growth_rows: 1, + viewport_top_y: 74, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(1), + effective_motion_rows_hint: Some(277), + }); + + assert!( + session + .should_fail_closed_outsized_observed_recovery_after_one_pixel_preview_local_commit( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 82, + motion_rows: 8, + mean_abs_diff_x100: 0, + }, + ) + ); + } + + #[test] + fn tiny_observed_burst_block_keeps_preview_local_baseline_stable() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 261; + session.observed_viewport_top_y = 261; + session.last_motion_rows_hint = Some(24); + session.transient_motion_rows_hint = Some(86); + session.transient_burst_search_enabled = true; + + session.refresh_preview_only_downward_local_sample( + &make_sparse_textlike_window(256, 120, 261), + session.stable_preview_only_downward_local_viewport_top_y(), ); + assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + session + .last_preview_only_downward_local_sample + .as_ref() + .map(|sample| sample.viewport_top_y), + Some(261) ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated + } + + #[test] + fn tiny_preview_only_local_recovery_fails_closed_during_large_transient_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(7); + session.transient_motion_rows_hint = Some(167); + session.transient_burst_search_enabled = true; + + assert!(session.should_fail_closed_tiny_preview_only_local_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 303, + motion_rows: 1, + mean_abs_diff_x100: 232, + } )); + } + + #[test] + fn tiny_preview_only_local_recovery_does_not_block_recorded_small_commit() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(1_217); + session.transient_burst_search_enabled = true; + + assert!(!session.should_fail_closed_tiny_preview_only_local_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 15, + motion_rows: 1, + mean_abs_diff_x100: 97, + } + )); + } - session.observed_viewport_top_y = 50; + #[test] + fn small_preview_only_local_recovery_lagging_recent_continuity_fails_closed_during_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(26); + session.transient_motion_rows_hint = Some(356); + session.transient_burst_search_enabled = true; + + assert!(session.should_fail_closed_tiny_preview_only_local_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 84, + motion_rows: 6, + mean_abs_diff_x100: 0, + } + )); + } - let height_before_return = session.export_image().height(); + #[test] + fn preview_only_local_tail_after_unresolved_burst_fails_closed() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 360; + session.last_block_reason = Some("no_downward_viewport_candidate_resolved"); + session.last_motion_rows_hint = Some(9); + session.transient_motion_rows_hint = Some(1_002); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 360), + viewport_top_y: 360, + }); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + assert!(session.should_fail_closed_preview_only_local_tail_after_unresolved_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 378, + motion_rows: 18, + mean_abs_diff_x100: 0, + } )); - assert_eq!(session.export_image().height(), height_before_return); - assert_eq!(session.observed_viewport_top_y, 2); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 3, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + } + + #[test] + fn preview_only_local_tail_after_unresolved_burst_does_not_block_without_extreme_gap() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 360; + session.last_block_reason = Some("no_downward_viewport_candidate_resolved"); + session.last_motion_rows_hint = Some(9); + session.transient_motion_rows_hint = Some(18); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 360), + viewport_top_y: 360, + }); + + assert!(!session.should_fail_closed_preview_only_local_tail_after_unresolved_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 378, + motion_rows: 18, + mean_abs_diff_x100: 0, + } + )); + } + + #[test] + fn preview_only_local_tail_after_unresolved_burst_does_not_block_after_registered_growth_matches_pending_band() + { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 184; + session.last_block_reason = Some("no_downward_viewport_candidate_resolved"); + session.last_motion_rows_hint = Some(1); + session.transient_motion_rows_hint = Some(277); + session.transient_burst_search_enabled = true; + session.pending_unresolved_burst_registered_growth_viewport_top_y = Some(461); + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 184), + viewport_top_y: 184, + }); + + assert!(!session.should_fail_closed_preview_only_local_tail_after_unresolved_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 186, + motion_rows: 2, + mean_abs_diff_x100: 125, + } + )); + } + + #[test] + fn exactly_corroborated_preview_local_tail_fails_closed_in_extreme_transient_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(10); + session.transient_motion_rows_hint = Some(1_057); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(20); + session.last_downward_viewport_candidates_before_prune = + Some("PreviewOnlyLocalSample@472/20:0,CommittedKeyframe@472/20:0".to_string()); + for (viewport_top_y, growth_rows) in [(442_i32, 8_u32), (452_i32, 10_u32)] { + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window( + 256, + 120, + u32::try_from(viewport_top_y).unwrap(), + ), + growth_rows, + viewport_top_y, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(growth_rows), + effective_motion_rows_hint: Some(1_057), + }); + } + + assert!( + session.should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 472, + motion_rows: 20, + mean_abs_diff_x100: 0, + }, + ) ); - assert_eq!(session.export_image().height(), height_before_return + 1); - assert_eq!(session.export_image().get_pixel(0, 7), &Rgba([80, 0, 0, 255])); } #[test] - fn resume_frontier_direct_proof_resumes_growth_across_skipped_anchor_undercount() { - let document = (0_u16..96) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + fn moderate_transient_preview_local_tail_is_not_blocked_by_extreme_burst_rule() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(20); + session.transient_motion_rows_hint = Some(110); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(24); + session.last_downward_viewport_candidates_before_prune = + Some("PreviewOnlyLocalSample@261/24:329,CommittedKeyframe@512/275:460".to_string()); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 237), + growth_rows: 20, + viewport_top_y: 237, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(20), + effective_motion_rows_hint: Some(110), + }); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 217), + growth_rows: 18, + viewport_top_y: 217, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(104), + }); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } + assert!( + !session.should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 261, + motion_rows: 24, + mean_abs_diff_x100: 329, + }, + ) ); + } - let height_before_rewind = session.export_image().height(); - let rewind_frame = make_window(&document, 3, 5, 48); + #[test] + fn burst_prefers_observed_sample_over_underconsumed_preview_only_local_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert!(matches!( - session.observe_upward_sample(rewind_frame).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange - )); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!(session.observed_viewport_top_y, 5); - assert!(session.resume_frontier_requires_reacquire); - assert_eq!( - session - .observe_downward_motion_while_resume_frontier_active( - make_window(&document, 3, 28, 48), - 14, - true, - ) - .unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 8 } + session.last_motion_rows_hint = Some(38); + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 120, motion_rows: 32 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 8 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + session.should_prefer_observed_sample_over_preview_only_local_recovery(primary, local) + ); + } + + #[test] + fn burst_keeps_preview_only_local_recovery_when_observed_is_only_modestly_ahead() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(38); + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 120, motion_rows: 16 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 8 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + !session.should_prefer_observed_sample_over_preview_only_local_recovery(primary, local) ); - assert_eq!(session.export_image().height(), height_before_rewind + 8); - assert_eq!(session.current_viewport_top_y, 28); - assert_eq!(session.observed_viewport_top_y, 28); - assert_eq!(session.resume_frontier_top_y, None); - assert!(!session.resume_frontier_requires_reacquire); - assert_eq!(session.export_image(), &make_test_image(3, &document[..76])); } #[test] - fn resume_frontier_exact_last_committed_reacquire_remains_valid_when_direct_proof_is_too_weak() - { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - [90, 0, 0, 255], - [100, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn tiny_recent_continuity_burst_prefers_preview_local_over_far_ahead_observed_sample() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(225); + session.transient_burst_search_enabled = true; - let height_before_resume = session.export_image().height(); - let exact_last_committed_frame = session.last_committed_frame.clone(); + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 12 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 405, motion_rows: 3 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; - session.resume_frontier_top_y = Some(session.current_viewport_top_y); - session.resume_frontier_requires_reacquire = true; - session.observed_viewport_top_y = session.current_viewport_top_y - 1; + assert!( + session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) + ); + } - let mut near_match = exact_last_committed_frame.clone(); + #[test] + fn tiny_recent_continuity_burst_does_not_force_preview_local_when_observed_is_still_nearby() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - paint_row(&mut near_match, 4, [111, 0, 0, 255]); + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(225); + session.transient_burst_search_enabled = true; - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active(near_match, 1, true) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.observed_viewport_top_y, 1); - assert!(session.resume_frontier_requires_reacquire); - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active( - exact_last_committed_frame, - 1, - true, - ) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.observed_viewport_top_y, 2); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert!(!session.resume_frontier_requires_reacquire); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 3, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 6 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 405, motion_rows: 3 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + !session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) ); - assert_eq!(session.export_image().height(), height_before_resume + 1); - assert_eq!(session.current_viewport_top_y, 3); - assert_eq!(session.observed_viewport_top_y, 3); - assert_eq!(session.resume_frontier_top_y, None); - assert_eq!(session.export_image().get_pixel(0, 7), &Rgba([80, 0, 0, 255])); } #[test] - fn resume_frontier_direct_match_can_append_before_observed_return_reaches_frontier() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - [90, 0, 0, 255], - [100, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn tiny_recent_continuity_burst_does_not_force_one_pixel_preview_local_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(1_211); + session.transient_burst_search_enabled = true; - session.observe_upward_rewind(2); + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 553, motion_rows: 413 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 97, motion_rows: 1 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; - assert_eq!(session.last_motion_rows_hint, None); - assert_eq!(session.observed_viewport_top_y, 0); - assert_eq!(session.resume_frontier_top_y, Some(2)); + assert!( + !session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) + ); + } - let height_before_resume = session.export_image().height(); + #[test] + fn repeated_missing_burst_frames_can_prefer_one_pixel_preview_local_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(277); + session.transient_burst_search_enabled = true; + session.consecutive_transient_burst_missing_downward_candidate_frames = 2; + + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 116 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 149, motion_rows: 1 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 3, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + assert!( + session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) ); - assert_eq!(session.export_image().height(), height_before_resume + 1); - assert_eq!(session.current_viewport_top_y, 3); - assert_eq!(session.observed_viewport_top_y, 3); - assert_eq!(session.resume_frontier_top_y, None); - assert!(!session.resume_frontier_requires_reacquire); - assert_eq!(session.export_image().get_pixel(0, 7), &Rgba([80, 0, 0, 255])); } #[test] - fn large_downward_step_after_rewind_only_appends_residual_new_rows() { - let document = (0_u16..96) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + fn preview_local_slowdown_followup_can_prefer_one_pixel_preview_local_recovery() { + let previous = make_sparse_textlike_window(256, 120, 16); + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.transient_motion_rows_hint = Some(29); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { + frame: previous, + growth_rows: 4, + viewport_top_y: 145, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(4), + effective_motion_rows_hint: Some(8), + }); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 41 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 410, motion_rows: 1 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) ); + } - let height_before_rewind = session.export_image().height(); + #[test] + fn preview_local_slowdown_followup_can_prefer_near_continuity_preview_local_recovery() { + let previous = make_sparse_textlike_window(256, 120, 16); + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(10); + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 416 }); + session.growth_history.push(GrowthCommit { + frame: previous, + growth_rows: 10, + viewport_top_y: 416, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(10), + effective_motion_rows_hint: Some(10), + }); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 5, 48)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange - )); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!(session.observed_viewport_top_y, 5); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_rewind); - assert_eq!(session.observed_viewport_top_y, 20); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 24, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 4 } + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 158 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 697, motion_rows: 12 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) ); - assert_eq!(session.export_image().height(), height_before_rewind + 4); - assert_eq!(session.current_viewport_top_y, 24); - assert_eq!(session.observed_viewport_top_y, 24); - assert_eq!(session.resume_frontier_top_y, None); - assert_eq!(session.export_image(), &make_test_image(3, &document[..72])); } #[test] - fn moving_resume_frontier_survives_two_rewind_resume_cycles_without_duplicate_growth() { - let document = (0_u16..128) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + fn preview_local_slowdown_followup_without_recent_small_preview_commit_does_not_prefer_local() { + let previous = make_sparse_textlike_window(256, 120, 16); + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.transient_motion_rows_hint = Some(29); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { + frame: previous, + growth_rows: 12, + viewport_top_y: 145, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(12), + effective_motion_rows_hint: Some(12), + }); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } + let primary = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 41 }, + source: DownwardSampleMatchSource::ObservedSample, + }; + let local = DownwardSampleMatch { + matched: DirectionMatch { mean_abs_diff_x100: 410, motion_rows: 1 }, + source: DownwardSampleMatchSource::PreviewOnlyLocalSample, + }; + + assert!( + !session.should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) ); + } - let height_after_initial_growth = session.export_image().height(); + #[test] + fn observed_burst_catch_up_commit_seeds_preview_local_baseline() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 5, 48)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + + assert!(session.should_seed_preview_only_local_after_observed_burst_commit( + "sample_motion_downward_growth_from_observed_keyframe", + 32, + Some(38), )); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!(session.observed_viewport_top_y, 5); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + } + + #[test] + fn non_observed_or_non_catch_up_commit_does_not_seed_preview_local_baseline() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.transient_motion_rows_hint = Some(1_150); + session.transient_burst_search_enabled = true; + + assert!(!session.should_seed_preview_only_local_after_observed_burst_commit( + "sample_motion_downward_growth_from_committed_keyframe", + 32, + Some(38), )); - assert_eq!(session.export_image().height(), height_after_initial_growth); - assert_eq!(session.current_viewport_top_y, 20); - assert_eq!(session.observed_viewport_top_y, 20); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 24, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 4 } - ); - assert_eq!(session.export_image().height(), height_after_initial_growth + 4); - assert_eq!(session.current_viewport_top_y, 24); - assert_eq!(session.observed_viewport_top_y, 24); - assert_eq!(session.resume_frontier_top_y, None); - assert_eq!(session.export_image(), &make_test_image(3, &document[..72])); + assert!(!session.should_seed_preview_only_local_after_observed_burst_commit( + "sample_motion_downward_growth_from_observed_keyframe", + 38, + Some(38), + )); + } + + #[test] + fn preview_local_burst_commit_preserves_local_baseline_for_next_frame() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - let height_after_first_resume = session.export_image().height(); + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 12, 48)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + assert!(session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 18, + Some(12), )); - assert_eq!(session.resume_frontier_top_y, Some(24)); - assert_eq!(session.observed_viewport_top_y, 12); - assert!(session.resume_frontier_requires_reacquire); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 24, 48)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + } + + #[test] + fn preview_local_burst_commit_does_not_preserve_local_baseline_for_tiny_or_far_growth() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; + + assert!(!session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 1, + Some(12), + )); + assert!(!session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 36, + Some(12), )); - assert_eq!(session.export_image().height(), height_after_first_resume); - assert_eq!(session.current_viewport_top_y, 24); - assert_eq!(session.observed_viewport_top_y, 24); - assert_eq!(session.resume_frontier_top_y, Some(24)); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 28, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 4 } - ); - assert_eq!(session.export_image().height(), height_after_first_resume + 4); - assert_eq!(session.current_viewport_top_y, 28); - assert_eq!(session.observed_viewport_top_y, 28); - assert_eq!(session.resume_frontier_top_y, None); - assert!(!session.resume_frontier_requires_reacquire); - assert_eq!(session.export_image(), &make_test_image(3, &document[..76])); } #[test] - fn blocked_frontier_crossing_requires_reacquire_before_later_valid_resume() { - let document = (0_u16..96) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 48), 320).unwrap(); + fn preview_local_non_burst_small_slowdown_preserves_local_baseline_for_next_frame() { + let session = ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 20, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } - ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 5, 48)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + assert!(session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 4, + Some(8), )); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!(session.observed_viewport_top_y, 5); + } - let unrelated_rows = vec![[250, 250, 250, 255]; 48]; - let unrelated_frame = make_test_image(3, &unrelated_rows); - let height_before_block = session.export_image().height(); + #[test] + fn preview_local_non_burst_tiny_or_growing_commit_does_not_preserve_local_baseline() { + let session = ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active(unrelated_frame, 19, true) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + assert!(!session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 1, + Some(8), )); - assert_eq!(session.export_image().height(), height_before_block); - assert_eq!(session.current_viewport_top_y, 20); - assert_eq!(session.observed_viewport_top_y, 19); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!(session.export_image(), &make_test_image(3, &document[..68])); - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active( - make_window(&document, 3, 20, 48), - 1, - true, - ) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + assert!(!session.should_preserve_preview_only_local_after_preview_only_burst_commit( + "sample_motion_downward_growth_from_preview_only_local_sample", + 10, + Some(8), )); - assert_eq!(session.export_image().height(), height_before_block); - assert_eq!(session.current_viewport_top_y, 20); - assert_eq!(session.observed_viewport_top_y, 20); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert_eq!( + } + + #[test] + fn corroborated_huge_local_jump_after_preview_local_commit_blocks_far_committed_only_recovery() + { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 230; + session.last_motion_rows_hint = Some(18); + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(164); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 230), + growth_rows: 18, + viewport_top_y: 230, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(226), + }); + + assert!( session - .observe_downward_motion_while_resume_frontier_active( - make_window(&document, 3, 28, 48), - 8, - true, + .should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }, + 164, ) - .unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 8 } ); - assert_eq!(session.current_viewport_top_y, 28); - assert_eq!(session.observed_viewport_top_y, 28); - assert_eq!(session.resume_frontier_top_y, None); - assert_eq!(session.export_image(), &make_test_image(3, &document[..76])); } #[test] - fn resume_frontier_blocked_downward_path_preserves_local_progress_without_upward_conflict() { - let document = (0_u16..160) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let width = 64; - let mut session = ScrollSession::new(make_window(&document, width, 0, 48), 320).unwrap(); + fn materially_smaller_observed_motion_still_blocks_huge_committed_only_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 230; + session.last_motion_rows_hint = Some(18); + session.transient_motion_rows_hint = Some(282); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(112); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(276); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 230), + growth_rows: 18, + viewport_top_y: 230, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(282), + }); - assert_eq!( - session.observe_downward_sample(make_window(&document, width, 20, 48)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 20 } + assert!( + session + .should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 506, + motion_rows: 276, + mean_abs_diff_x100: 0, + }, + 276, + ) ); + } - let height_before_resume = session.export_image().height(); + #[test] + fn nearby_committed_recovery_is_not_blocked_when_local_jump_is_not_huge() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 230; + session.last_motion_rows_hint = Some(18); + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(38); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(38); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 230), + growth_rows: 18, + viewport_top_y: 230, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(226), + }); - session.resume_frontier_top_y = Some(20); - session.resume_frontier_requires_reacquire = true; - session.observed_viewport_top_y = 36; - session.last_sample_frame = make_window(&document, width, 36, 48); - session.last_sample_fingerprint = - Some(scroll_capture::scroll_capture_fingerprint(&session.last_sample_frame)); + assert!( + !session + .should_fail_closed_far_committed_only_recovery_after_corroborated_huge_local_jump( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 268, + motion_rows: 38, + mean_abs_diff_x100: 0, + }, + 38, + ) + ); + } - let resume_frame = make_window(&document, width, 52, 48); + #[test] + fn suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed_fails_closed() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 230; + session.last_motion_rows_hint = Some(18); + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 230), + growth_rows: 18, + viewport_top_y: 230, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(226), + }); - assert_eq!( + assert!( session - .evaluate_reference_overlap_direction_preferred_only( - &session.last_sample_frame, - &resume_frame, - ScrollDirection::Down, - Some(16), + .should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }), + &[DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }], ) - .map(|matched| matched.motion_rows), - Some(16) ); - assert_eq!( - session - .evaluate_reference_overlap_direction_preferred_only( - &session.last_committed_frame, - &resume_frame, - ScrollDirection::Down, - Some(16), + } + + #[test] + fn suppressed_preview_local_jump_without_exact_committed_corroboration_stays_unblocked() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 230; + session.last_motion_rows_hint = Some(18); + session.transient_motion_rows_hint = Some(226); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 230), + growth_rows: 18, + viewport_top_y: 230, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(18), + effective_motion_rows_hint: Some(226), + }); + + assert!( + !session + .should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }), + &[DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 398, + motion_rows: 186, + mean_abs_diff_x100: 0, + }], + ) + ); + } + + #[test] + fn committed_followup_after_suppressed_huge_preview_local_jump_fails_closed() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.transient_burst_search_enabled = true; + session.last_preview_only_local_registration_result = Some("no_match"); + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(164); + + assert!( + session.should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }), + &[ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 398, + motion_rows: 186, + mean_abs_diff_x100: 0, + }, + ], + ) + ); + } + + #[test] + fn committed_followup_without_pending_suppressed_preview_local_jump_stays_unblocked() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.transient_burst_search_enabled = true; + session.last_preview_only_local_registration_result = Some("no_match"); + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(164); + + assert!( + !session + .should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + None, + &[DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }], ) - .map(|matched| matched.motion_rows), - None ); - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active(resume_frame, 16, true,) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.current_viewport_top_y, 20); - assert_eq!(session.observed_viewport_top_y, 52); - assert_eq!(session.resume_frontier_top_y, Some(20)); - assert!(session.resume_frontier_requires_reacquire); } #[test] - fn resume_frontier_requires_reacquire_before_ambiguous_match_can_reach_frontier() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [10, 0, 0, 255], - [20, 0, 0, 255], - [10, 0, 0, 255], - [20, 0, 0, 255], - [10, 0, 0, 255], - [20, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn committed_followup_after_extreme_preview_local_tail_block_fails_closed() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); + session.transient_burst_search_enabled = true; - session.observe_upward_rewind(1); + assert!( + session.should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 472, + motion_rows: 20, + mean_abs_diff_x100: 0, + }), + &[ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 472, + motion_rows: 20, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 472, + motion_rows: 30, + mean_abs_diff_x100: 0, + }, + ], + ) + ); + } - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.observed_viewport_top_y, 1); + #[test] + fn committed_followup_after_extreme_preview_local_tail_block_ignores_non_exact_match() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - let height_before_resume = session.export_image().height(); + session.transient_burst_search_enabled = true; - assert!(matches!( - session - .observe_downward_motion_while_resume_frontier_active( - make_window(&document, 3, 1, 5), - 1, - true, - ) - .unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.current_viewport_top_y, 2); - assert_eq!(session.observed_viewport_top_y, 1); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert!(session.resume_frontier_requires_reacquire); + assert!( + !session.should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 472, + motion_rows: 20, + mean_abs_diff_x100: 0, + }), + &[DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 472, + motion_rows: 30, + mean_abs_diff_x100: 0, + }], + ) + ); } #[test] - fn resume_frontier_blocks_same_viewport_small_mutation_without_new_growth() { - let document = [ - [0, 0, 0, 255], - [0, 0, 0, 255], - [0, 0, 0, 255], - [0, 0, 0, 255], - [40, 0, 0, 255], - [0, 0, 0, 255], - [40, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn suppressed_huge_preview_local_followup_block_budget_scales_with_far_recovery_ratio() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + session.last_motion_rows_hint = Some(18); assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + session.suppressed_huge_preview_only_local_followup_block_budget(Some( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }, + )), + 5 ); assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } + session.suppressed_huge_preview_only_local_followup_block_budget(Some( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 290, + motion_rows: 42, + mean_abs_diff_x100: 0, + }, + )), + 3 ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange - )); + } - let height_before_resume = session.export_image().height(); - let mut same_viewport_with_small_mutation = make_window(&document, 3, 2, 5); + #[test] + fn huge_suppressed_jump_window_refreshes_observed_baseline_without_advancing_viewport() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - paint_row(&mut same_viewport_with_small_mutation, 3, [16, 0, 0, 255]); + assert!(!session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); - assert!(matches!( - session.observe_downward_sample(same_viewport_with_small_mutation).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.current_viewport_top_y, 2); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert_eq!(session.export_image(), &make_test_image(3, &document)); + session.pending_suppressed_huge_preview_only_local_followup = + Some(DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 394, + motion_rows: 164, + mean_abs_diff_x100: 0, + }); + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); + + session.pending_suppressed_huge_preview_only_local_followup = None; + session.blocked_followup_after_suppressed_huge_preview_local_jump = true; + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); + + session.blocked_followup_after_suppressed_huge_preview_local_jump = false; + session.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); } #[test] - fn first_small_upward_rewind_after_large_downward_commit_arms_and_blocks_duplicate_return() { - let document = (0_u16..220) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 64), 320).unwrap(); + fn huge_far_committed_block_resets_preview_only_local_baseline() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 40, 64)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 40 } + session.refresh_preview_only_downward_local_sample( + &make_sparse_textlike_window(256, 120, 32), + Some(32), ); - assert_eq!(session.last_motion_rows_hint, Some(40)); + assert!(session.last_preview_only_downward_local_sample.is_some()); + assert!(!session.should_reset_preview_only_local_baseline_after_huge_far_committed_block()); - let height_before_rewind = session.export_image().height(); + session.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; + assert!(session.should_reset_preview_only_local_baseline_after_huge_far_committed_block()); + } - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 35, 64)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + #[test] + fn seeded_preview_only_local_catch_up_candidate_can_commit_small_tail_growth() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 162; + session.seeded_preview_only_local_after_observed_burst_commit = true; + + assert!(session.seeded_preview_only_local_catch_up_candidate_can_commit( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 170, + motion_rows: 8, + mean_abs_diff_x100: 0, + } )); - assert_eq!(session.resume_frontier_top_y, Some(40)); - - let observed_after_first_rewind = session.observed_viewport_top_y; + } - assert!(observed_after_first_rewind < 40); - assert!(session.resume_frontier_requires_reacquire); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 40, 64)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated + #[test] + fn unseeded_preview_only_local_candidate_still_needs_normal_burst_rules() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 162; + + assert!(!session.seeded_preview_only_local_catch_up_candidate_can_commit( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, + viewport_top_y: 170, + motion_rows: 8, + mean_abs_diff_x100: 0, + } )); - assert_eq!(session.export_image().height(), height_before_rewind); - assert_eq!(session.current_viewport_top_y, 40); - assert_eq!(session.observed_viewport_top_y, 40); - assert_eq!(session.resume_frontier_top_y, Some(40)); } #[test] - fn upward_preview_change_without_overlap_proof_fails_closed_into_rewind_block() { - let document = [ - [10, 0, 0, 255], - [20, 0, 0, 255], - [30, 0, 0, 255], - [40, 0, 0, 255], - [50, 0, 0, 255], - [60, 0, 0, 255], - [70, 0, 0, 255], - [80, 0, 0, 255], - [90, 0, 0, 255], - ]; - let mut session = ScrollSession::new(make_window(&document, 3, 0, 5), 320).unwrap(); + fn seeded_preview_only_local_recovery_range_includes_one_pixel_tail_growth() { + let previous = make_sparse_textlike_window(256, 120, 0); + let next = make_sparse_textlike_window(256, 120, 1); + let mut session = ScrollSession::new(previous.clone(), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.seeded_preview_only_local_after_observed_burst_commit = true; + + let range = session + .preview_only_local_recovery_motion_range( + &previous, + &next, + OverlapSearchConfig::default(), + ) + .unwrap(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } - ); + assert_eq!(*range.start(), 1); + assert_eq!(*range.end(), 6); + } - let height_before_rewind = session.export_image().height(); - let unrelated_frame = make_test_image( - 3, - &[ - [200, 200, 0, 255], - [0, 200, 200, 255], - [200, 0, 200, 255], - [20, 20, 20, 255], - [240, 240, 240, 255], - ], - ); + #[test] + fn unseeded_preview_only_local_recovery_range_keeps_hint_floor() { + let previous = make_sparse_textlike_window(256, 120, 0); + let next = make_sparse_textlike_window(256, 120, 1); + let mut session = ScrollSession::new(previous.clone(), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + + let range = session + .preview_only_local_recovery_motion_range( + &previous, + &next, + OverlapSearchConfig::default(), + ) + .unwrap(); - assert_eq!( - session.observe_upward_sample(unrelated_frame).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - ); - assert_eq!(session.export_image().height(), height_before_rewind); - assert_eq!(session.resume_frontier_top_y, Some(2)); - assert!(session.resume_frontier_requires_reacquire); - assert!(session.observed_viewport_top_y < 2); - assert!(matches!( - session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(), - ScrollObserveOutcome::NoChange | ScrollObserveOutcome::PreviewUpdated - )); - assert_eq!(session.export_image().height(), height_before_rewind); - assert_eq!(session.current_viewport_top_y, 2); - assert_eq!(session.resume_frontier_top_y, Some(2)); + assert_eq!(*range.start(), 2); + assert_eq!(*range.end(), 6); } #[test] - fn rewind_active_upward_override_prefers_smaller_committed_match() { - let sample = DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 240 }; - let committed = DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 96 }; + fn preview_local_slowdown_followup_range_allows_one_pixel_tail_in_burst() { + let previous = make_sparse_textlike_window(256, 120, 16); + let next = make_sparse_textlike_window(256, 120, 17); + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.transient_motion_rows_hint = Some(29); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { + frame: previous.clone(), + growth_rows: 4, + viewport_top_y: 145, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(4), + effective_motion_rows_hint: Some(8), + }); - assert_eq!( - scroll_capture::rewind_active_upward_override_match( - Some(sample), - Some(committed), - true - ), - Some((committed, true)) - ); - assert_eq!( - scroll_capture::rewind_active_upward_override_match( - Some(sample), - Some(committed), - false - ), - None - ); + let range = session + .preview_only_local_recovery_motion_range( + &previous, + &next, + OverlapSearchConfig::default(), + ) + .unwrap(); + + assert_eq!(*range.start(), 1); + assert_eq!(*range.end(), 6); } #[test] - fn rewind_active_repeated_upward_prefers_conservative_committed_match() { - let document = (0_u16..220) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 64), 320).unwrap(); + fn preview_local_followup_without_recent_small_preview_commit_keeps_hint_floor_in_burst() { + let previous = make_sparse_textlike_window(256, 120, 16); + let next = make_sparse_textlike_window(256, 120, 17); + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.transient_motion_rows_hint = Some(29); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = + Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { + frame: previous.clone(), + growth_rows: 12, + viewport_top_y: 145, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(12), + effective_motion_rows_hint: Some(12), + }); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 40, 64)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 40 } - ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 35, 64)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange - )); - assert_eq!(session.resume_frontier_top_y, Some(40)); + let range = session + .preview_only_local_recovery_motion_range( + &previous, + &next, + OverlapSearchConfig::default(), + ) + .unwrap(); - let observed_after_first_rewind = session.observed_viewport_top_y; + assert_eq!(*range.start(), 2); + assert_eq!(*range.end(), 6); + } - assert!(observed_after_first_rewind < 40); + #[test] + fn tiny_committed_keyframe_recovery_fails_closed_during_large_transient_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 68; + session.last_motion_rows_hint = Some(6); + session.transient_motion_rows_hint = Some(401); + session.transient_burst_search_enabled = true; + + assert!(session.should_fail_closed_tiny_committed_keyframe_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 70, + motion_rows: 12, + mean_abs_diff_x100: 654, + } + )); + } - let stale_lower_sample = make_window(&document, 3, 116, 64); + #[test] + fn tiny_committed_keyframe_recovery_does_not_block_meaningful_growth_during_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 68; + session.last_motion_rows_hint = Some(6); + session.transient_motion_rows_hint = Some(401); + session.transient_burst_search_enabled = true; + + assert!(!session.should_fail_closed_tiny_committed_keyframe_recovery_in_burst( + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 81, + motion_rows: 23, + mean_abs_diff_x100: 696, + } + )); + } - session.last_sample_frame = stale_lower_sample.clone(); - session.last_sample_fingerprint = - Some(super::scroll_capture_fingerprint(&stale_lower_sample)); - session.last_motion_rows_hint = Some(48); + #[test] + fn underconsumed_observed_recovery_fails_closed_when_nearby_committed_candidate_reaches_recent_continuity() + { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(20); + session.transient_motion_rows_hint = Some(75); + session.transient_burst_search_enabled = true; + + let candidates_before_prune = vec![ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 289, + motion_rows: 8, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 289, + motion_rows: 8, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 291, + motion_rows: 30, + mean_abs_diff_x100: 0, + }, + ]; + let candidates_after_prune = vec![ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 289, + motion_rows: 8, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 289, + motion_rows: 8, + mean_abs_diff_x100: 0, + }, + ]; - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 20, 64)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + assert!(session.should_fail_closed_underconsumed_observed_recovery_in_burst( + &candidates_before_prune, + &candidates_after_prune, )); - assert_eq!(session.resume_frontier_top_y, Some(40)); - assert!(session.observed_viewport_top_y <= observed_after_first_rewind); - assert!(session.resume_frontier_requires_reacquire); } #[test] - fn rewind_active_fail_closed_upward_path_preserves_later_resume_growth() { - let document = (0_u16..240) - .map(|row| { - [((row * 17) % 251) as u8, ((row * 47) % 251) as u8, ((row * 89) % 251) as u8, 255] - }) - .collect::>(); - let mut session = ScrollSession::new(make_window(&document, 3, 0, 64), 320).unwrap(); + fn underconsumed_observed_recovery_does_not_block_small_recorded_burst_commit() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(4); + session.transient_motion_rows_hint = Some(466); + session.transient_burst_search_enabled = true; + + let candidates_before_prune = vec![ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::ObservedSample, + viewport_top_y: 14, + motion_rows: 2, + mean_abs_diff_x100: 6, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 14, + motion_rows: 2, + mean_abs_diff_x100: 6, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 14, + motion_rows: 6, + mean_abs_diff_x100: 16, + }, + ]; + let candidates_after_prune = candidates_before_prune[..2].to_vec(); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 32, 64)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 32 } - ); - assert_eq!( - session.observe_downward_sample(make_window(&document, 3, 64, 64)).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 32 } - ); - assert!(matches!( - session.observe_upward_sample(make_window(&document, 3, 16, 64)).unwrap(), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - | ScrollObserveOutcome::PreviewUpdated - | ScrollObserveOutcome::NoChange + assert!(!session.should_fail_closed_underconsumed_observed_recovery_in_burst( + &candidates_before_prune, + &candidates_after_prune, )); - assert_eq!(session.resume_frontier_top_y, Some(64)); - assert!(session.resume_frontier_requires_reacquire); + } - let observed_before_conflict = session.observed_viewport_top_y; + #[test] + fn low_confidence_committed_only_recovery_without_local_anchor_fails_closed_during_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 134; + session.last_motion_rows_hint = Some(43); + session.transient_motion_rows_hint = Some(1_142); + session.transient_burst_search_enabled = true; + + let candidates = vec![ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 190, + motion_rows: 56, + mean_abs_diff_x100: 621, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 157, + motion_rows: 73, + mean_abs_diff_x100: 557, + }, + ]; - assert!(observed_before_conflict < 64); + assert!(session.should_fail_closed_far_committed_only_recovery_without_local_anchor( + candidates[1], + &candidates, + )); + } + + #[test] + fn small_continuity_preview_local_registration_blocks_larger_committed_only_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 62; + session.last_motion_rows_hint = Some(2); + session.transient_motion_rows_hint = Some(225); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 31), + viewport_top_y: 62, + }); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(3); + + let candidates = vec![ + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 74, + motion_rows: 12, + mean_abs_diff_x100: 0, + }, + DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 78, + motion_rows: 14, + mean_abs_diff_x100: 0, + }, + ]; - let stale_lower_sample = make_window(&document, 3, 128, 64); + assert!(session.should_fail_closed_far_committed_only_recovery_without_local_anchor( + candidates[0], + &candidates, + )); + } - session.last_sample_frame = stale_lower_sample.clone(); - session.last_sample_fingerprint = - Some(super::scroll_capture_fingerprint(&stale_lower_sample)); - session.last_motion_rows_hint = Some(64); + #[test] + fn suppressed_large_preview_local_registration_blocks_underconsumed_committed_only_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 202; + session.last_motion_rows_hint = Some(8); + session.transient_motion_rows_hint = Some(575); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 31), + viewport_top_y: 202, + }); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(272); - let height_before_resume = session.export_image().height(); - let resumed_target_top_y = 80_i32; - let resumed_motion_rows = - u32::try_from(resumed_target_top_y - observed_before_conflict).unwrap(); + let candidates = vec![DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 220, + motion_rows: 32, + mean_abs_diff_x100: 765, + }]; - assert_eq!( - session.arm_unconfirmed_upward_rewind( - &make_window(&document, 3, 0, 64), - super::scroll_capture_fingerprint(&make_window(&document, 3, 0, 64)), - Some(MotionObservation { direction: ScrollDirection::Up, motion_rows: 64 }), - false, - "scroll_capture.rewind_armed_without_match", - "rewind_active_upward_input_conflicted_with_last_committed_downward_match", - ), - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - ); - assert_eq!(session.export_image().height(), height_before_resume); - assert_eq!(session.resume_frontier_top_y, Some(64)); - assert_eq!(session.observed_viewport_top_y, observed_before_conflict); - assert!(session.resume_frontier_requires_reacquire); - assert_eq!(session.last_sample_frame, stale_lower_sample); - assert_eq!( + assert!( session - .observe_downward_motion_while_resume_frontier_active( - make_window(&document, 3, resumed_target_top_y as usize, 64), - resumed_motion_rows, - true, + .should_fail_closed_underconsumed_committed_only_recovery_after_suppressed_preview_local_match( + candidates[0], + session.growth_rows_for_candidate_viewport_top_y(candidates[0].viewport_top_y), ) - .unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 16 } ); - assert_eq!(session.export_image().height(), height_before_resume + 16); - assert_eq!(session.current_viewport_top_y, 80); - assert_eq!(session.observed_viewport_top_y, 80); - assert_eq!(session.resume_frontier_top_y, None); - assert!(!session.resume_frontier_requires_reacquire); - assert_eq!(session.export_image(), &make_test_image(3, &document[..144])); } #[test] - fn trustworthy_committed_upward_match_overrides_non_upward_sample_motion() { - let up = DirectionMatch { mean_abs_diff_x100: 120, motion_rows: 18 }; - let down = DirectionMatch { mean_abs_diff_x100: 90, motion_rows: 18 }; + fn corroborated_sample_registrations_block_committed_only_recovery_without_viewport_anchor() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 237; + session.last_motion_rows_hint = Some(20); + session.transient_motion_rows_hint = Some(145); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 237), + viewport_top_y: 237, + }); + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(135); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(116); + + let preferred = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 353, + motion_rows: 116, + mean_abs_diff_x100: 0, + }; - assert_eq!(scroll_capture::preferred_upward_override_match(Some(up), Some(down)), Some(up)); - assert_eq!(scroll_capture::preferred_upward_override_match(Some(up), None), Some(up)); + assert!(session + .should_fail_closed_committed_only_recovery_after_corroborated_sample_registration_without_viewport_anchor( + preferred, + session.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y), + )); } #[test] - fn downward_input_requires_committed_growth_before_confirming_upward_match() { - let up = DirectionMatch { mean_abs_diff_x100: 120, motion_rows: 18 }; + fn corroborated_sample_registrations_block_older_keyframe_recovery_by_growth_band() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 237; + session.last_motion_rows_hint = Some(20); + session.transient_motion_rows_hint = Some(249); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 237), + viewport_top_y: 237, + }); + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(258); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(180); + + let preferred = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 464, + motion_rows: 271, + mean_abs_diff_x100: 700, + }; - assert_eq!( - scroll_capture::upward_confirmation_match_for_downward_input(Some(up), None, false), - None - ); - assert_eq!( - scroll_capture::upward_confirmation_match_for_downward_input(Some(up), None, true), - Some(up) - ); + assert!(session + .should_fail_closed_committed_only_recovery_after_corroborated_sample_registration_without_viewport_anchor( + preferred, + session.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y), + )); } #[test] - fn downward_input_upward_confirmation_requires_direction_margin_when_downward_exists() { - let weak_up = DirectionMatch { mean_abs_diff_x100: 120, motion_rows: 18 }; - let strong_down = DirectionMatch { mean_abs_diff_x100: 90, motion_rows: 18 }; - let strong_up = DirectionMatch { mean_abs_diff_x100: 40, motion_rows: 18 }; - let weak_down = DirectionMatch { mean_abs_diff_x100: 160, motion_rows: 18 }; + fn observed_burst_outpacing_recent_preview_local_commit_blocks_committed_only_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 237; + session.last_motion_rows_hint = Some(20); + session.transient_motion_rows_hint = Some(145); + session.transient_burst_search_enabled = true; + session.last_observed_sample_registration_result = Some("matched"); + session.last_observed_sample_registration_motion_rows = Some(135); + session.last_preview_only_local_registration_result = Some("no_match"); + session.growth_history.push(super::GrowthCommit { + frame: make_sparse_textlike_window(256, 120, 237), + growth_rows: 20, + viewport_top_y: 237, + decision_source: DownwardViewportCandidateSource::PreviewOnlyLocalSample + .decision_source(), + detected_motion_rows: Some(20), + effective_motion_rows_hint: Some(145), + }); - assert_eq!( - scroll_capture::upward_confirmation_match_for_downward_input( - Some(weak_up), - Some(strong_down), - true, - ), - None - ); - assert_eq!( - scroll_capture::upward_confirmation_match_for_downward_input( - Some(strong_up), - Some(weak_down), - true, - ), - Some(strong_up) - ); + let preferred = DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 353, + motion_rows: 116, + mean_abs_diff_x100: 0, + }; + + assert!(session + .should_fail_closed_committed_only_recovery_when_observed_burst_outpaces_recent_preview_local_commit( + preferred, + session.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y), + )); } #[test] - fn upward_input_prefers_conservative_committed_override_before_rewind_active() { - let sample = DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 288 }; - let committed = DirectionMatch { mean_abs_diff_x100: 0, motion_rows: 48 }; - - assert_eq!( - scroll_capture::preferred_upward_input_override_match(Some(sample), Some(committed)), - Some((committed, true)) - ); - assert_eq!( - scroll_capture::preferred_upward_input_override_match(Some(committed), Some(sample)), - Some((committed, false)) + fn suppressed_large_preview_local_registration_helper_skips_hint_band_committed_recovery() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.current_viewport_top_y = 202; + session.last_motion_rows_hint = Some(8); + session.transient_motion_rows_hint = Some(575); + session.transient_burst_search_enabled = true; + session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { + frame: make_sparse_textlike_window(256, 120, 31), + viewport_top_y: 202, + }); + session.last_preview_only_local_registration_result = Some("matched"); + session.last_preview_only_local_registration_motion_rows = Some(272); + + let candidates = vec![DownwardViewportCandidate { + source: DownwardViewportCandidateSource::CommittedKeyframe, + viewport_top_y: 500, + motion_rows: 310, + mean_abs_diff_x100: 0, + }]; + + assert!( + !session + .should_fail_closed_underconsumed_committed_only_recovery_after_suppressed_preview_local_match( + candidates[0], + session.growth_rows_for_candidate_viewport_top_y(candidates[0].viewport_top_y), + ) ); } #[test] - fn weak_committed_upward_match_does_not_override_non_upward_sample_motion() { - let up = DirectionMatch { mean_abs_diff_x100: 500, motion_rows: 18 }; - let down = DirectionMatch { mean_abs_diff_x100: 90, motion_rows: 18 }; - - assert_eq!(scroll_capture::preferred_upward_override_match(Some(up), Some(down)), None); - assert_eq!(scroll_capture::preferred_upward_override_match(Some(up), None), None); + fn weak_tiny_committed_keyframe_match_retries_full_range_during_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(14); + session.transient_motion_rows_hint = Some(380); + session.transient_burst_search_enabled = true; + + assert!(session.should_retry_committed_keyframe_registration_across_full_range( + DownwardRegistration::Matched(DirectionMatch { + mean_abs_diff_x100: 733, + motion_rows: 7, + }), + )); + assert_eq!( + session.prefer_full_range_committed_keyframe_registration( + DownwardRegistration::Matched(DirectionMatch { + mean_abs_diff_x100: 733, + motion_rows: 7, + }), + DownwardRegistration::Matched(DirectionMatch { + mean_abs_diff_x100: 0, + motion_rows: 50, + }), + ), + DownwardRegistration::Matched(DirectionMatch { + mean_abs_diff_x100: 0, + motion_rows: 50, + }), + ); } #[test] - fn rewind_active_sample_only_upward_conflict_with_committed_downward_fails_closed() { - let sample_up = DirectionMatch { mean_abs_diff_x100: 80, motion_rows: 64 }; - let committed_down = DirectionMatch { mean_abs_diff_x100: 80, motion_rows: 80 }; - let committed_up = DirectionMatch { mean_abs_diff_x100: 70, motion_rows: 32 }; - - assert!(scroll_capture::rewind_active_upward_motion_should_fail_closed( - Some(sample_up), - None, - Some(committed_down), - true, - )); - assert!(!scroll_capture::rewind_active_upward_motion_should_fail_closed( - Some(sample_up), - Some(committed_up), - Some(committed_down), - true, - )); - assert!(!scroll_capture::rewind_active_upward_motion_should_fail_closed( - Some(sample_up), - None, - Some(committed_down), - false, + fn modest_committed_keyframe_match_does_not_retry_full_range_during_burst() { + let mut session = + ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + + session.last_motion_rows_hint = Some(9); + session.transient_motion_rows_hint = Some(1_284); + session.transient_burst_search_enabled = true; + + assert!(!session.should_retry_committed_keyframe_registration_across_full_range( + DownwardRegistration::Matched(DirectionMatch { + mean_abs_diff_x100: 301, + motion_rows: 27, + }), )); } diff --git a/packages/rsnap-overlay/src/worker.rs b/packages/rsnap-overlay/src/worker.rs index 2d818004..6e4650b2 100644 --- a/packages/rsnap-overlay/src/worker.rs +++ b/packages/rsnap-overlay/src/worker.rs @@ -14,7 +14,6 @@ use crate::backend::CaptureBackend; use crate::png; #[cfg(not(target_os = "macos"))] use crate::state::LiveCursorSample; -#[cfg(any(not(target_os = "macos"), test))] use crate::state::RectPoints; use crate::state::{GlobalPoint, MonitorRect, WindowHit, WindowListSnapshot}; @@ -45,7 +44,6 @@ pub(crate) enum WorkerRequest { monitor: MonitorRect, target: FreezeCaptureTarget, }, - #[cfg(not(target_os = "macos"))] CaptureMonitorRegion { monitor: MonitorRect, rect_px: RectPoints, @@ -94,11 +92,9 @@ pub(crate) enum WorkerErrorSource { EncodePng, FreezeCapture, RefreshWindowList, - #[cfg(any(not(target_os = "macos"), test))] CaptureMonitorRegion, } -#[cfg(any(not(target_os = "macos"), test))] #[derive(Debug)] pub(crate) enum CapturedMonitorRegionResult { Image(RgbaImage), @@ -111,7 +107,6 @@ pub(crate) enum WorkerRequestSendError { Disconnected, } -#[cfg(any(not(target_os = "macos"), test))] #[derive(Debug)] pub(crate) struct CapturedMonitorRegionResponse { pub(crate) monitor: MonitorRect, @@ -125,7 +120,6 @@ pub(crate) struct OverlayWorker { resp_rx: Receiver, #[cfg(all(test, target_os = "macos"))] debug_id: u64, - #[cfg(any(not(target_os = "macos"), test))] region_capture_resp_rx: Receiver, } impl OverlayWorker { @@ -135,18 +129,10 @@ impl OverlayWorker { ) -> Self { let (req_tx, req_rx) = mpsc::sync_channel(64); let (resp_tx, resp_rx) = mpsc::channel(); - #[cfg(any(not(target_os = "macos"), test))] let (region_capture_resp_tx, region_capture_resp_rx) = mpsc::channel(); thread::spawn(move || { - Self::run_worker_loop( - backend, - req_rx, - resp_tx, - #[cfg(any(not(target_os = "macos"), test))] - region_capture_resp_tx, - response_waker, - ) + Self::run_worker_loop(backend, req_rx, resp_tx, region_capture_resp_tx, response_waker) }); Self { @@ -154,7 +140,6 @@ impl OverlayWorker { resp_rx, #[cfg(all(test, target_os = "macos"))] debug_id: next_worker_debug_id(), - #[cfg(any(not(target_os = "macos"), test))] region_capture_resp_rx, } } @@ -163,9 +148,7 @@ impl OverlayWorker { mut backend: Box, req_rx: Receiver, resp_tx: Sender, - #[cfg(any(not(target_os = "macos"), test))] region_capture_resp_tx: Sender< - CapturedMonitorRegionResponse, - >, + region_capture_resp_tx: Sender, response_waker: Option>, ) { while let Ok(first) = req_rx.recv() { @@ -180,7 +163,6 @@ impl OverlayWorker { pending.dispatch( &mut *backend, &resp_tx, - #[cfg(any(not(target_os = "macos"), test))] ®ion_capture_resp_tx, response_waker.as_deref(), ); @@ -282,7 +264,6 @@ impl OverlayWorker { } } - #[cfg(any(not(target_os = "macos"), test))] fn handle_capture_monitor_region_request( backend: &mut dyn CaptureBackend, resp_tx: &Sender, @@ -392,7 +373,6 @@ impl OverlayWorker { } } - #[cfg(any(not(target_os = "macos"), test))] fn send_region_capture_response( resp_tx: &Sender, response_waker: Option<&(dyn Fn() + Send + Sync)>, @@ -468,7 +448,6 @@ impl OverlayWorker { } } - #[cfg(not(target_os = "macos"))] pub(crate) fn request_capture_monitor_region( &self, monitor: MonitorRect, @@ -492,7 +471,6 @@ impl OverlayWorker { self.debug_id } - #[cfg(any(not(target_os = "macos"), test))] pub(crate) fn try_recv_captured_monitor_region(&self) -> Option { match self.region_capture_resp_rx.try_recv() { Ok(msg) => Some(msg), @@ -508,7 +486,6 @@ struct PendingWorkerRequests { last_sample_cursor: Option<(MonitorRect, GlobalPoint, u64, bool, u32, u32)>, last_refresh_window_list: bool, last_freeze: Option<(MonitorRect, FreezeCaptureTarget)>, - #[cfg(not(target_os = "macos"))] last_capture_region: Option<(MonitorRect, RectPoints, u64)>, last_encode: Option, } @@ -536,7 +513,6 @@ impl PendingWorkerRequests { WorkerRequest::FreezeCapture { monitor, target } => { self.last_freeze = Some((monitor, target)); }, - #[cfg(not(target_os = "macos"))] WorkerRequest::CaptureMonitorRegion { monitor, rect_px, request_id } => { self.last_capture_region = Some((monitor, rect_px, request_id)); }, @@ -550,9 +526,7 @@ impl PendingWorkerRequests { self, backend: &mut dyn CaptureBackend, resp_tx: &Sender, - #[cfg(any(not(target_os = "macos"), test))] _region_capture_resp_tx: &Sender< - CapturedMonitorRegionResponse, - >, + _region_capture_resp_tx: &Sender, response_waker: Option<&(dyn Fn() + Send + Sync)>, ) { if let Some(image) = self.last_encode { @@ -565,7 +539,6 @@ impl PendingWorkerRequests { return; } - #[cfg(not(target_os = "macos"))] if let Some((monitor, rect_px, request_id)) = self.last_capture_region { OverlayWorker::handle_capture_monitor_region_request( backend, diff --git a/scripts/scroll-capture-smoke-macos.sh b/scripts/scroll-capture-smoke-macos.sh deleted file mode 100755 index 29914f59..00000000 --- a/scripts/scroll-capture-smoke-macos.sh +++ /dev/null @@ -1,545 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Deterministic macOS desktop smoke for rsnap scroll capture mode. -# -# Assumptions: -# - Runs inside a logged-in macOS GUI session. -# - Screen Recording is already granted for the rsnap binary/build command so -# live capture frames are available. -# - Accessibility is only needed for the smoke automation (`osascript`/`swift`) -# and optional scrollbar verification. It is not a runtime prerequisite for -# rsnap scroll-capture availability. -# - The script intentionally launches a fresh rsnap process and closes open -# TextEdit windows so the run is reproducible. -# - Top-level settle delays default to zero on release builds; environment -# overrides remain available if a local GUI session needs extra slack. -# - The content movement must come from synthetic scroll-wheel input. The -# script may read the TextEdit vertical scrollbar value for optional -# verification, but it never drives the scrollbar through Accessibility -# writes. -# - The required assertion surface is log-based: startup, overlay start, -# scroll-capture start, and at least one growth commit. The harness uses the rsnap -# tray `Capture` menu item instead of synthetic global-hotkey injection -# because the tray path is more stable in automated GUI sessions. Scrollbar -# verification is an optional stronger check when AX access is available. - -usage() { - cat <<'EOF' -Usage: scroll-capture-smoke-macos.sh [--self-check] [--help] - -Environment overrides: - RSNAP_CMD command used to launch rsnap (default: target/release/rsnap - when present, else cargo run --release -p rsnap) - RSNAP_RUST_LOG log filter for the rsnap process - (default: rsnap=info,rsnap_overlay=trace) - TEXTEDIT_BOUNDS "left,top,right,bottom" for the fixture window - DRAG_START "x,y" capture drag start point - DRAG_END "x,y" capture drag end point - SCROLL_POINT "x,y" point inside the capture rect for downward scrolls - SCROLL_EVENTS number of downward wheel events to emit - SCROLL_DELTA negative pixel delta for each wheel event - VERIFY_SCROLLBAR set to 1 to require AX scrollbar verification, 0 to skip - it, or auto to use it only when available (default: auto) - OVERLAY_SETTLE_S delay after overlay startup before drag selection - (default: 0) - DRAG_SETTLE_S delay after drag selection before entering scroll capture - mode (default: 0) - SCROLL_MODE_SETTLE_S - delay after scroll capture start before emitting scroll - events (default: 0) - WAIT_STARTUP_S timeout for startup log marker - WAIT_OVERLAY_S timeout for overlay start log marker - WAIT_SCROLL_CAPTURE_S - timeout for scroll capture mode start marker - WAIT_APPEND_S timeout for append marker -EOF -} - -self_check() { - if [[ "$(uname -s)" != "Darwin" ]]; then - echo "scroll-capture smoke is macOS-only" >&2 - return 1 - fi - - for cmd in osascript swift python3 rg; do - if ! command -v "$cmd" >/dev/null 2>&1; then - echo "missing required tool: $cmd" >&2 - return 1 - fi - done - - echo "[smoke] self-check ok" -} - -case "${1:-}" in - --help|-h) - usage - exit 0 - ;; - --self-check) - self_check - exit $? - ;; - "") - ;; - *) - usage >&2 - exit 2 - ;; -esac - -self_check - -ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -LOG_DIR="$HOME/Library/Application Support/ink.hack.rsnap/logs" -DEFAULT_RSNAP_CMD="cargo run --release -p rsnap" -if [[ -x "$ROOT_DIR/target/release/rsnap" ]]; then - DEFAULT_RSNAP_CMD="$ROOT_DIR/target/release/rsnap" -fi -RSNAP_CMD="${RSNAP_CMD:-$DEFAULT_RSNAP_CMD}" -RSNAP_RUST_LOG="${RSNAP_RUST_LOG:-rsnap=info,rsnap_overlay=trace}" -TEXTEDIT_BOUNDS="${TEXTEDIT_BOUNDS:-120,120,1040,960}" -DRAG_START="${DRAG_START:-}" -DRAG_END="${DRAG_END:-}" -SCROLL_POINT="${SCROLL_POINT:-}" -SCROLL_EVENTS="${SCROLL_EVENTS:-28}" -SCROLL_DELTA="${SCROLL_DELTA:--32}" -OVERLAY_SETTLE_S="${OVERLAY_SETTLE_S:-0}" -DRAG_SETTLE_S="${DRAG_SETTLE_S:-0}" -SCROLL_MODE_SETTLE_S="${SCROLL_MODE_SETTLE_S:-0}" -WAIT_STARTUP_S="${WAIT_STARTUP_S:-30}" -WAIT_OVERLAY_S="${WAIT_OVERLAY_S:-10}" -WAIT_SCROLL_CAPTURE_S="${WAIT_SCROLL_CAPTURE_S:-10}" -WAIT_APPEND_S="${WAIT_APPEND_S:-10}" -VERIFY_SCROLLBAR="${VERIFY_SCROLLBAR:-auto}" -SCROLL_CAPTURE_MODE_ATTEMPTS="${SCROLL_CAPTURE_MODE_ATTEMPTS:-3}" -SCROLL_CAPTURE_MODE_ATTEMPT_WAIT_S="${SCROLL_CAPTURE_MODE_ATTEMPT_WAIT_S:-2}" - -RSNAP_LOG="" -RSNAP_PID="" -SCROLLBAR_VERIFICATION_ACTIVE=0 -SCROLLBAR_AUTO_PROBE_PENDING=0 -SCROLL_CAPTURE_GROWTH_PATTERN='op="scroll_capture\.(appended|committed)"' -FIXTURE_FILE_BASE="$(mktemp -t rsnap-scroll-capture-fixture)" -FIXTURE_FILE="${FIXTURE_FILE_BASE}.txt" -mv "$FIXTURE_FILE_BASE" "$FIXTURE_FILE" -SWIFT_HELPER_BASE="$(mktemp -t rsnap-scroll-capture-swift)" -SWIFT_HELPER="${SWIFT_HELPER_BASE}.swift" -mv "$SWIFT_HELPER_BASE" "$SWIFT_HELPER" - -close_textedit_windows() { - osascript <<'APPLESCRIPT' >/dev/null 2>&1 || true -try - tell application "TextEdit" - if it is running then - repeat with docRef in (documents as list) - try - close docRef saving no - end try - end repeat - try - close every window saving no - end try - end if - end tell -end try -APPLESCRIPT -} - -stop_existing_rsnap() { - if ! pgrep -x rsnap >/dev/null 2>&1; then - return - fi - - echo "[smoke] stopping existing rsnap processes" >&2 - pkill -x rsnap >/dev/null 2>&1 || true - - local deadline=$((SECONDS + 10)) - while pgrep -x rsnap >/dev/null 2>&1; do - if (( SECONDS > deadline )); then - fail "existing rsnap process did not stop" - fi - sleep 0.2 - done -} - -cleanup() { - if [[ -n "$RSNAP_PID" ]] && kill -0 "$RSNAP_PID" >/dev/null 2>&1; then - kill "$RSNAP_PID" >/dev/null 2>&1 || true - wait "$RSNAP_PID" >/dev/null 2>&1 || true - fi - - close_textedit_windows - rm -f "$FIXTURE_FILE" "$SWIFT_HELPER" -} -trap cleanup EXIT - -fail() { - echo "[smoke] $*" >&2 - if [[ -n "$RSNAP_LOG" && -f "$RSNAP_LOG" ]]; then - echo "[smoke] recent rsnap log excerpt:" >&2 - tail -n 80 "$RSNAP_LOG" >&2 || true - fi - exit 1 -} - -refresh_log_path() { - RSNAP_LOG="$(ls -1t "$LOG_DIR"/rsnap*.log 2>/dev/null | head -n 1 || true)" -} - -wait_for_pattern() { - local pattern="$1" - local timeout_s="$2" - local deadline=$((SECONDS + timeout_s)) - - while (( SECONDS <= deadline )); do - refresh_log_path - if [[ -n "$RSNAP_LOG" && -f "$RSNAP_LOG" ]] && rg -q "$pattern" "$RSNAP_LOG"; then - return 0 - fi - sleep 0.25 - done - - return 1 -} - -configure_scrollbar_verification() { - case "$VERIFY_SCROLLBAR" in - 1|true|yes|on) - SCROLLBAR_VERIFICATION_ACTIVE=1 - ;; - 0|false|no|off) - SCROLLBAR_VERIFICATION_ACTIVE=0 - ;; - auto|"") - SCROLLBAR_VERIFICATION_ACTIVE=0 - SCROLLBAR_AUTO_PROBE_PENDING=1 - ;; - *) - fail "invalid VERIFY_SCROLLBAR value: $VERIFY_SCROLLBAR" - ;; - esac -} - -maybe_capture_scrollbar_value() { - local value="" - - if (( SCROLLBAR_VERIFICATION_ACTIVE )); then - value="$(read_textedit_scrollbar_value)" - printf '%s\n' "$value" - return 0 - fi - - if (( SCROLLBAR_AUTO_PROBE_PENDING )); then - SCROLLBAR_AUTO_PROBE_PENDING=0 - if ! value="$(read_textedit_scrollbar_value 2>/dev/null)"; then - return 0 - fi - - SCROLLBAR_VERIFICATION_ACTIVE=1 - printf '%s\n' "$value" - return 0 - fi - - return 0 -} - -scrollbar_value_increased() { - local initial="$1" - local final="$2" - python3 - "$initial" "$final" <<'PY' -import sys - -initial = float(sys.argv[1]) -final = float(sys.argv[2]) -sys.exit(0 if final > initial else 1) -PY -} - -write_fixture() { - python3 - "$FIXTURE_FILE" <<'PY' -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -with path.open("w") as handle: - for i in range(1, 701): - handle.write(f"Line {i:03d} -- rsnap scroll capture smoke fixture with deterministic text.\n") -PY -} - -open_fixture_in_textedit() { - local bounds="$1" - local left top right bottom - IFS=, read -r left top right bottom <<<"$bounds" - close_textedit_windows - open -a TextEdit "$FIXTURE_FILE" - osascript <&2 - sleep 0.2 - ((attempt++)) - done - - return 1 -} - -cat > "$SWIFT_HELPER" <<'SWIFT' -import Cocoa -import ApplicationServices - -func readPoint(_ key: String) -> CGPoint { - let value = ProcessInfo.processInfo.environment[key] ?? "" - let parts = value.split(separator: ",") - guard parts.count == 2, - let x = Double(parts[0]), - let y = Double(parts[1]) else { - fputs("invalid point env for \(key): \(value)\n", stderr) - exit(2) - } - return CGPoint(x: x, y: y) -} - -func readInt(_ key: String) -> Int { - guard let raw = ProcessInfo.processInfo.environment[key], let value = Int(raw) else { - fputs("invalid int env for \(key)\n", stderr) - exit(2) - } - return value -} - -func sleepMs(_ ms: useconds_t) { usleep(ms * 1000) } - -func mouseEvent(_ type: CGEventType, at point: CGPoint, button: CGMouseButton = .left) { - let src = CGEventSource(stateID: .hidSystemState) - let event = CGEvent(mouseEventSource: src, mouseType: type, mouseCursorPosition: point, mouseButton: button) - event?.post(tap: .cghidEventTap) -} - -func drag(from start: CGPoint, to end: CGPoint, steps: Int) { - mouseEvent(.mouseMoved, at: start) - sleepMs(120) - mouseEvent(.leftMouseDown, at: start) - sleepMs(120) - for step in 1...steps { - let t = CGFloat(step) / CGFloat(steps) - let point = CGPoint(x: start.x + (end.x - start.x) * t, y: start.y + (end.y - start.y) * t) - mouseEvent(.leftMouseDragged, at: point) - sleepMs(25) - } - mouseEvent(.leftMouseUp, at: end) -} - -func scroll(at point: CGPoint, deltaY: Int32, times: Int) { - mouseEvent(.mouseMoved, at: point) - sleepMs(120) - for _ in 0../tmp/rsnap-scroll-capture-smoke-rsnap.out 2>&1 & - RSNAP_PID=$! -} - -mkdir -p "$LOG_DIR" -stop_existing_rsnap -rm -f "$LOG_DIR"/rsnap*.log -write_fixture -launch_rsnap -configure_scrollbar_verification -wait_for_pattern 'Starting rsnap\.' "$WAIT_STARTUP_S" || fail "rsnap did not log startup" -open_fixture_in_textedit "$TEXTEDIT_BOUNDS" -ACTUAL_TEXTEDIT_BOUNDS="$(read_textedit_front_window_bounds | tr -d ' ')" -resolve_capture_points "$ACTUAL_TEXTEDIT_BOUNDS" -INITIAL_SCROLLBAR_VALUE="$(maybe_capture_scrollbar_value | tr -d ' ')" -echo "[smoke] textedit bounds: $ACTUAL_TEXTEDIT_BOUNDS" -echo "[smoke] drag start: $DRAG_START" -echo "[smoke] drag end: $DRAG_END" -echo "[smoke] scroll point: $SCROLL_POINT" -if (( SCROLLBAR_VERIFICATION_ACTIVE )); then - echo "[smoke] initial scrollbar value: $INITIAL_SCROLLBAR_VALUE" -else - echo "[smoke] scrollbar verification: skipped" -fi -trigger_capture_from_tray_menu -wait_for_pattern 'Capture overlay started\.' "$WAIT_OVERLAY_S" || fail "capture overlay did not start" -sleep "$OVERLAY_SETTLE_S" -MODE=drag START_POINT="$DRAG_START" END_POINT="$DRAG_END" DRAG_STEPS=28 swift "$SWIFT_HELPER" -sleep "$DRAG_SETTLE_S" -enter_scroll_capture_mode || fail "scroll capture mode did not start" -sleep "$SCROLL_MODE_SETTLE_S" -MODE=scroll SCROLL_POINT="$SCROLL_POINT" SCROLL_DELTA="$SCROLL_DELTA" SCROLL_EVENTS="$SCROLL_EVENTS" swift "$SWIFT_HELPER" -FINAL_SCROLLBAR_VALUE="$(maybe_capture_scrollbar_value | tr -d ' ')" -if (( SCROLLBAR_VERIFICATION_ACTIVE )); then - echo "[smoke] final scrollbar value: $FINAL_SCROLLBAR_VALUE" - scrollbar_value_increased "$INITIAL_SCROLLBAR_VALUE" "$FINAL_SCROLLBAR_VALUE" \ - || fail "synthetic scroll input did not move the TextEdit vertical scrollbar" -fi -wait_for_pattern "$SCROLL_CAPTURE_GROWTH_PATTERN" "$WAIT_APPEND_S" \ - || fail "scroll capture did not record any committed growth" - -echo "[smoke] PASS" -if [[ -n "$RSNAP_LOG" ]]; then - rg -n "Starting rsnap\\.|Capture overlay started\\.|op=\"scroll_capture.start\"|$SCROLL_CAPTURE_GROWTH_PATTERN" "$RSNAP_LOG" -fi -press_escape >/dev/null 2>&1 || true From 786d4c65554b161ccaa23ed66d72671b3ecce41b Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 15:11:11 +0800 Subject: [PATCH 02/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture","summary":"stabilize macOS worker-pairwise live stitching","intent":"reduce duplicate-frame storms and trace overhead in downward touchpad capture","impact":"improves live scroll capture continuity and keeps release candidate stable under fresh touchpad smoke","breaking":false,"risk":"medium","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"}]} --- packages/rsnap-overlay/src/overlay.rs | 65 ++++++++ .../src/overlay/scroll_runtime.rs | 58 ++++++- .../src/overlay/trace_recording.rs | 146 ++++++++++++++++-- packages/rsnap-overlay/src/scroll_capture.rs | 6 +- 4 files changed, 252 insertions(+), 23 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 25748700..9e189a83 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -303,6 +303,8 @@ const SCROLL_PREVIEW_WINDOW_WIDTH_POINTS: f64 = 260.0; const SCROLL_PREVIEW_WINDOW_HEIGHT_POINTS: f64 = 360.0; const SCROLL_PREVIEW_WINDOW_MARGIN_POINTS: i32 = 16; const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); +const SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL: Duration = + Duration::from_millis(60); #[cfg(target_os = "macos")] const SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE: Duration = Duration::from_millis(180); const SCROLL_CAPTURE_PREVIEW_WIDTH_PX: u32 = 320; @@ -17589,6 +17591,69 @@ mod tests { assert_eq!(scroll_capture_export_height(&session), 724); } + #[cfg(target_os = "macos")] + #[test] + fn maybe_tick_scroll_capture_worker_path_backs_off_after_duplicate_committed_frame() { + let monitor = test_monitor(); + let rect = RectPoints::new(100, 120, 512, 640); + let base = make_browser_like_worker_capture_window(512, 640, 0); + let step_one = make_browser_like_worker_capture_window(512, 640, 84); + let step_two = make_browser_like_worker_capture_window(512, 640, 168); + let mut session = OverlaySession::new(); + + session.worker = Some(OverlayWorker::new( + Box::new(SequenceScrollCaptureBackend::new([ + Some(step_one.clone()), + Some(step_one), + Some(step_two), + ])), + None, + )); + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); + + set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + assert_eq!(scroll_capture_export_height(&session), 724); + + session.scroll_capture.last_external_scroll_input_seq = 2; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); + assert_eq!(scroll_capture_export_height(&session), 724); + + session.maybe_tick_scroll_capture(); + assert!( + session.scroll_capture.inflight_request_id.is_none(), + "duplicate committed worker frame should back off instead of immediately re-requesting" + ); + + session.scroll_capture.last_external_scroll_input_seq = 3; + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(Instant::now()); + session.scroll_capture.input_gesture_active = true; + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 168); + assert_eq!(scroll_capture_export_height(&session), 808); + } + #[cfg(target_os = "macos")] #[test] fn handle_scroll_input_ready_drains_input_and_polls_stream_fallback() { diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index 6dcb78cb..8d3db240 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -1,9 +1,10 @@ -use std::time::Instant; +use std::time::{Duration, Instant}; use color_eyre::Result; use image::RgbaImage; use crate::overlay::SCROLL_CAPTURE_SAMPLE_INTERVAL; +use crate::overlay::SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL; #[cfg(target_os = "macos")] use crate::overlay::session_state::ScrollCaptureLiveFrame; #[cfg(target_os = "macos")] @@ -155,6 +156,33 @@ impl OverlaySession { ); } + #[cfg(target_os = "macos")] + fn schedule_backoff_scroll_capture_worker_retry_if_fresh_downward_input( + &mut self, + now: Instant, + why: &'static str, + delay: Duration, + ) { + let fresh_downward_input = self.scroll_capture.input_direction + == Some(ScrollDirection::Down) + && self.scroll_capture.input_direction_at.is_some_and(|input_direction_at| { + now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS + }); + if !fresh_downward_input { + return; + } + + self.scroll_capture.next_sample_at = Some(now + delay); + tracing::info!( + op = "scroll_capture.worker_retry_scheduled_with_backoff", + reason = why, + delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX), + last_external_scroll_input_seq = self.scroll_capture.last_external_scroll_input_seq, + downward_motion_rows_pending = self.scroll_capture.downward_motion_rows_pending, + "Scheduled a delayed worker retry because fresh downward input was still active but the latest worker frame repeated the committed content." + ); + } + #[cfg(target_os = "macos")] pub(super) fn try_consume_scroll_stream_frame(&mut self) -> bool { let Some(monitor) = self.scroll_capture.monitor else { @@ -345,7 +373,7 @@ impl OverlaySession { let prior_direction = self.scroll_capture.input_direction; let prior_gesture_active = self.scroll_capture.input_gesture_active; - tracing::info!( + tracing::debug!( op = "scroll_capture.replayed_input", seq, prior_seq = self.scroll_capture.last_external_scroll_input_seq, @@ -391,7 +419,7 @@ impl OverlaySession { self.request_redraw_scroll_preview_window(); } - tracing::info!( + tracing::debug!( op = "scroll_capture.replayed_input_result", seq, recorded_at_ms_behind_pairing = u64::try_from( @@ -855,6 +883,8 @@ impl OverlaySession { ) { match outcome { Ok(ScrollObserveOutcome::NoChange) => { + let last_block_reason = + self.scroll_capture.session.as_ref().and_then(ScrollSession::last_block_reason); tracing::info!( op = "scroll_capture.frame_observed", frame_source = source.as_str(), @@ -863,21 +893,35 @@ impl OverlaySession { frame_px = ?frame_px, input_direction = ?self.scroll_capture.input_direction, input_gesture_active = self.scroll_capture.input_gesture_active, + last_block_reason = ?last_block_reason, export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), "Scroll-capture observed a frame but kept session state unchanged." ); if let Some(request_id) = source.worker_request_id() { #[cfg(target_os = "macos")] - self.schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( - Instant::now(), - "worker_no_change", - ); + { + let now = Instant::now(); + match last_block_reason { + Some("frame_matches_last_committed_frame") => self + .schedule_backoff_scroll_capture_worker_retry_if_fresh_downward_input( + now, + "worker_duplicate_committed_frame", + SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL, + ), + _ => self + .schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( + now, + "worker_no_change", + ), + } + } tracing::info!( op = "scroll_capture.worker_frame_processed", request_id, outcome = "no_change", frame_px = ?frame_px, input_direction = ?self.scroll_capture.input_direction, + last_block_reason = ?last_block_reason, "Worker-fed scroll-capture frame reached the session without changing preview or export state." ); } diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index 45a3c377..38460f88 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -13,12 +13,15 @@ use serde::{Deserialize, Serialize}; use super::{MonitorRect, RectPoints, ScrollCaptureFrameSource}; use crate::{ png, - scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}, + scroll_capture::{ + scroll_capture_fingerprint, ScrollDirection, ScrollObserveOutcome, ScrollSession, + }, }; const SCROLL_CAPTURE_TRACE_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE"; const SCROLL_CAPTURE_TRACE_DIR_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE_DIR"; const SCROLL_CAPTURE_TRACE_SCHEMA: &str = "scroll_capture_live_trace/1"; +const SCROLL_CAPTURE_TRACE_MANIFEST_FLUSH_INTERVAL: Duration = Duration::from_millis(250); #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ScrollCaptureLiveTraceManifest { @@ -233,7 +236,10 @@ pub(crate) struct ScrollCaptureTraceRecorder { trace_dir: PathBuf, manifest_path: PathBuf, started_at: Instant, + last_manifest_flush_at: Instant, next_frame_index: u64, + last_recorded_frame_fingerprint: Option>, + last_recorded_frame_path: Option, manifest: ScrollCaptureLiveTraceManifest, } @@ -301,22 +307,39 @@ impl ScrollCaptureTraceRecorder { snapshot_after: record.snapshot_after, }, )); - self.flush_manifest_best_effort("record_input"); + self.flush_manifest_if_due_best_effort("record_input"); } pub(crate) fn record_frame_observation(&mut self, record: ScrollCaptureTraceFrameRecord<'_>) { - let frame_index = self.next_frame_index; - self.next_frame_index = self.next_frame_index.saturating_add(1); - let frame_path = format!("frames/frame-{frame_index:06}.png"); + let frame_fingerprint = scroll_capture_fingerprint(record.frame); + let frame_path = if self + .last_recorded_frame_fingerprint + .as_ref() + .is_some_and(|previous| previous == &frame_fingerprint) + { + self.last_recorded_frame_path.clone().unwrap_or_else(|| { + let frame_index = self.next_frame_index; + self.next_frame_index = self.next_frame_index.saturating_add(1); + format!("frames/frame-{frame_index:06}.png") + }) + } else { + let frame_index = self.next_frame_index; + self.next_frame_index = self.next_frame_index.saturating_add(1); + let frame_path = format!("frames/frame-{frame_index:06}.png"); - if let Err(err) = self.write_frame(record.frame, &frame_path) { - tracing::warn!( - op = "scroll_capture.trace_write_frame_failed", - error = %err, - frame_index, - "Failed to persist scroll-capture trace frame." - ); - } + if let Err(err) = self.write_frame(record.frame, &frame_path) { + tracing::warn!( + op = "scroll_capture.trace_write_frame_failed", + error = %err, + frame_index, + "Failed to persist scroll-capture trace frame." + ); + } + self.last_recorded_frame_fingerprint = Some(frame_fingerprint); + self.last_recorded_frame_path = Some(frame_path.clone()); + + frame_path + }; let outcome = match record.outcome { Ok(value) => ScrollCaptureTraceRecordedOutcome::from(*value), @@ -335,7 +358,7 @@ impl ScrollCaptureTraceRecorder { outcome, }, )); - self.flush_manifest_best_effort("record_frame"); + self.flush_manifest_if_due_best_effort("record_frame"); } pub(crate) fn record_error(&mut self, message: &str) { @@ -415,9 +438,20 @@ impl ScrollCaptureTraceRecorder { final_error: None, finalized: false, }; - let recorder = Self { trace_dir, manifest_path, started_at, next_frame_index: 0, manifest }; + let mut recorder = Self { + trace_dir, + manifest_path, + started_at, + last_manifest_flush_at: started_at, + next_frame_index: 0, + last_recorded_frame_fingerprint: None, + last_recorded_frame_path: None, + manifest, + }; recorder.write_frame(base_frame, &recorder.manifest.base_frame_path)?; + recorder.last_recorded_frame_fingerprint = Some(scroll_capture_fingerprint(base_frame)); + recorder.last_recorded_frame_path = Some(recorder.manifest.base_frame_path.clone()); recorder.flush_manifest_best_effort("init"); Ok(recorder) @@ -448,6 +482,18 @@ impl ScrollCaptureTraceRecorder { } } + fn flush_manifest_if_due_best_effort(&mut self, op: &'static str) { + let now = Instant::now(); + if now.saturating_duration_since(self.last_manifest_flush_at) + < SCROLL_CAPTURE_TRACE_MANIFEST_FLUSH_INTERVAL + { + return; + } + + self.last_manifest_flush_at = now; + self.flush_manifest_best_effort(op); + } + fn flush_manifest(&self) -> Result<()> { let bytes = serde_json::to_vec_pretty(&self.manifest) .wrap_err("failed to serialize scroll-capture trace manifest")?; @@ -736,4 +782,74 @@ mod tests { assert!(loaded.resolve_frame_path("frames/final-export.png").exists()); assert!(loaded.manifest.final_snapshot.is_some()); } + + #[test] + fn trace_recorder_reuses_png_path_for_consecutive_identical_frames() { + let rows = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + ]; + let base_frame = make_window(&rows, 0); + let next_frame = make_window(&rows, 1); + let root = temp_trace_root(); + let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( + root, + test_monitor(), + test_rect(), + 320, + &base_frame, + ) + .unwrap(); + let start = Instant::now(); + let manifest_path = recorder.manifest_path().to_path_buf(); + let snapshot = ScrollCaptureTraceSessionSnapshot::capture( + None, + Some([next_frame.width(), next_frame.height()]), + Some(ScrollDirection::Down), + true, + 32.0, + Some(1), + ); + + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: &next_frame, + source: ScrollCaptureFrameSource::Worker { request_id: 1 }, + allow_stale_input: false, + prior_block_reason: None, + observed_at: start + Duration::from_millis(10), + snapshot_after: snapshot.clone(), + outcome: &Ok(ScrollObserveOutcome::PreviewUpdated), + }); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { + frame: &next_frame, + source: ScrollCaptureFrameSource::Worker { request_id: 2 }, + allow_stale_input: false, + prior_block_reason: Some("frame_matches_last_committed_frame"), + observed_at: start + Duration::from_millis(20), + snapshot_after: snapshot, + outcome: &Ok(ScrollObserveOutcome::NoChange), + }); + drop(recorder); + + let loaded = LoadedScrollCaptureLiveTrace::load(&manifest_path).unwrap(); + let entries = loaded + .manifest + .entries + .iter() + .filter_map(|entry| match entry { + ScrollCaptureLiveTraceEntry::Frame(frame) => Some(frame), + ScrollCaptureLiveTraceEntry::Input(_) => None, + }) + .collect::>(); + + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].frame_path, entries[1].frame_path); + assert!(loaded.resolve_frame_path(&entries[0].frame_path).exists()); + assert!(!loaded.resolve_frame_path("frames/frame-000001.png").exists()); + } } diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 671325aa..765f80a5 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -1420,7 +1420,7 @@ impl ScrollSession { block_reason: Option<&'static str>, ) { self.last_block_reason = block_reason; - tracing::info!( + tracing::debug!( op, input_direction = ?input_direction, detected_direction = ?detected_motion.map(|motion| motion.direction), @@ -3158,6 +3158,10 @@ impl ScrollSession { self.export_image.dimensions() } + pub(crate) fn last_block_reason(&self) -> Option<&'static str> { + self.last_block_reason + } + pub(crate) fn commit_telemetry(&self) -> ScrollCommitTelemetry { let last_commit = self.growth_history.last(); From bfa8a086618c9672848d8fcb97e1445eeb2d5998 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 15:19:50 +0800 Subject: [PATCH 03/19] {"schema":"delivery/1","type":"fix","scope":"ci","summary":"format rsnap-overlay cargo manifest for TOML checks","intent":"satisfy taplo formatting so PR CI goes green without changing runtime behavior","impact":"Language Checks/TOML checks can pass on PR #47 while scroll capture behavior remains unchanged","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"}]} --- packages/rsnap-overlay/Cargo.toml | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/rsnap-overlay/Cargo.toml b/packages/rsnap-overlay/Cargo.toml index 9ca3c4ee..0820ac01 100644 --- a/packages/rsnap-overlay/Cargo.toml +++ b/packages/rsnap-overlay/Cargo.toml @@ -16,22 +16,22 @@ path = "src/lib.rs" cargo-clippy = [] [dependencies] -arboard = { workspace = true } -color-eyre = { workspace = true } -directories = { workspace = true } -egui = { workspace = true } -egui-phosphor = { workspace = true } -egui-wgpu = { workspace = true } -egui-winit = { workspace = true } -image = { workspace = true } -pollster = { workspace = true } -serde = { workspace = true } -serde_json = "1.0" -thiserror = { workspace = true } -tracing = { workspace = true } +arboard = { workspace = true } +color-eyre = { workspace = true } +directories = { workspace = true } +egui = { workspace = true } +egui-phosphor = { workspace = true } +egui-wgpu = { workspace = true } +egui-winit = { workspace = true } +image = { workspace = true } +pollster = { workspace = true } +serde = { workspace = true } +serde_json = "1.0" +thiserror = { workspace = true } +tracing = { workspace = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -wgpu = { workspace = true } -winit = { workspace = true } +wgpu = { workspace = true } +winit = { workspace = true } [target.'cfg(not(target_os = "macos"))'.dependencies] device_query = { workspace = true } @@ -49,7 +49,7 @@ objc2-core-media = { workspace = true } objc2-core-video = { workspace = true } objc2-foundation = { workspace = true } objc2-screen-capture-kit = { workspace = true } -objc2-vision = "0.3.2" +objc2-vision = "0.3.2" raw-window-handle = { workspace = true } [dev-dependencies] From 8062a1e816a9001555f47e1dfd0af4cc4dcda835 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 15:54:11 +0800 Subject: [PATCH 04/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-review","summary":"repair CI and review feedback for scroll capture","intent":"restore coalesced live-frame wakeups and reduce hot-path scroll logging noise","impact":"prevents overlay stream wake storms on the app thread and keeps touchpad input telemetry out of info-level logs","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- apps/rsnap/src/app.rs | 14 +- apps/rsnap/src/app/capture.rs | 13 +- apps/rsnap/src/app/runtime.rs | 2 + .../src/app/scroll_input_macos/decode.rs | 2 +- .../rsnap/src/app/scroll_input_macos/state.rs | 43 +- .../examples/scroll_capture_replay.rs | 40 +- packages/rsnap-overlay/src/backend.rs | 9 +- packages/rsnap-overlay/src/lib.rs | 1 - .../src/live_frame_stream_macos.rs | 321 ++-- packages/rsnap-overlay/src/overlay.rs | 179 ++- .../src/overlay/replay_support.rs | 400 +++-- .../src/overlay/scroll_runtime.rs | 434 +++-- .../src/overlay/session_state.rs | 15 +- .../src/overlay/trace_recording.rs | 129 +- packages/rsnap-overlay/src/scroll_capture.rs | 1426 +++++++++-------- 15 files changed, 1763 insertions(+), 1265 deletions(-) diff --git a/apps/rsnap/src/app.rs b/apps/rsnap/src/app.rs index 9e56b926..406c00ea 100644 --- a/apps/rsnap/src/app.rs +++ b/apps/rsnap/src/app.rs @@ -6,7 +6,10 @@ mod scroll_input_macos; mod shell; #[cfg(target_os = "macos")] -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; use color_eyre::eyre::Result; use global_hotkey::{GlobalHotKeyEvent, GlobalHotKeyManager, hotkey::HotKey}; @@ -74,6 +77,8 @@ struct App { #[cfg(target_os = "macos")] scroll_input_shared_state: Arc, #[cfg(target_os = "macos")] + overlay_stream_event_pending: Arc, + #[cfg(target_os = "macos")] startup_permissions_checked: bool, } impl App { @@ -121,10 +126,17 @@ impl App { #[cfg(target_os = "macos")] scroll_input_shared_state, #[cfg(target_os = "macos")] + overlay_stream_event_pending: Arc::new(AtomicBool::new(false)), + #[cfg(target_os = "macos")] startup_permissions_checked: false, } } + #[cfg(target_os = "macos")] + fn finish_coalesced_overlay_stream_frame_send(&self) { + self.overlay_stream_event_pending.store(false, Ordering::Release); + } + fn open_settings_window(&mut self, event_loop: &ActiveEventLoop, requested_by: &'static str) { if let Some(window) = self.settings_window.as_ref() { tracing::info!(requested_by = %requested_by, "Settings already open; focusing."); diff --git a/apps/rsnap/src/app/capture.rs b/apps/rsnap/src/app/capture.rs index 4833fc5d..549a53ba 100644 --- a/apps/rsnap/src/app/capture.rs +++ b/apps/rsnap/src/app/capture.rs @@ -1,5 +1,6 @@ #[cfg(target_os = "macos")] use std::sync::Arc; +use std::sync::atomic::Ordering; #[cfg(target_os = "macos")] use std::time::Duration; @@ -105,8 +106,11 @@ impl App { let mut overlay_session = OverlaySession::with_config(self.overlay_config()); + #[cfg(target_os = "macos")] + self.finish_coalesced_overlay_stream_frame_send(); #[cfg(target_os = "macos")] self.scroll_input_shared_state.clear(); + #[cfg(target_os = "macos")] self.scroll_input_shared_state.set_event_waker(Some(Arc::new({ let overlay_proxy = self.overlay_proxy.clone(); @@ -115,13 +119,18 @@ impl App { let _ = overlay_proxy.send_event(UserEvent::OverlayScrollInput); } }))); - #[cfg(target_os = "macos")] overlay_session.set_scroll_frame_waker(Arc::new({ let overlay_proxy = self.overlay_proxy.clone(); + let overlay_stream_event_pending = Arc::clone(&self.overlay_stream_event_pending); move || { - let _ = overlay_proxy.send_event(UserEvent::OverlayStreamFrame); + if overlay_stream_event_pending + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() && overlay_proxy.send_event(UserEvent::OverlayStreamFrame).is_err() + { + overlay_stream_event_pending.store(false, Ordering::Release); + } } })); #[cfg(target_os = "macos")] diff --git a/apps/rsnap/src/app/runtime.rs b/apps/rsnap/src/app/runtime.rs index 041e8d2d..885b2788 100644 --- a/apps/rsnap/src/app/runtime.rs +++ b/apps/rsnap/src/app/runtime.rs @@ -41,6 +41,8 @@ impl ApplicationHandler for App { UserEvent::TrayIcon => {}, #[cfg(target_os = "macos")] UserEvent::OverlayStreamFrame => { + self.finish_coalesced_overlay_stream_frame_send(); + if let Some(session) = self.overlay_session.as_mut() { let control = session.handle_scroll_stream_frame_ready(); diff --git a/apps/rsnap/src/app/scroll_input_macos/decode.rs b/apps/rsnap/src/app/scroll_input_macos/decode.rs index 7d393952..33f0c3de 100644 --- a/apps/rsnap/src/app/scroll_input_macos/decode.rs +++ b/apps/rsnap/src/app/scroll_input_macos/decode.rs @@ -43,7 +43,7 @@ pub(super) fn decode_scroll_input_from_cg_event( let gesture_ended = scroll_phase_bits_are_terminal(scroll_phase) || scroll_phase_bits_are_terminal(momentum_phase); - tracing::info!( + tracing::debug!( op = "scroll_input.tap_decoded", raw_delta_y, global_x = location.x, diff --git a/apps/rsnap/src/app/scroll_input_macos/state.rs b/apps/rsnap/src/app/scroll_input_macos/state.rs index 8830a068..dc410beb 100644 --- a/apps/rsnap/src/app/scroll_input_macos/state.rs +++ b/apps/rsnap/src/app/scroll_input_macos/state.rs @@ -5,6 +5,8 @@ use std::sync::{ }; use std::time::{Duration, Instant}; +type SharedScrollInputEventWaker = Arc; + const SHARED_SCROLL_INPUT_QUEUE_CAPACITY: usize = 512; #[derive(Default)] @@ -115,7 +117,7 @@ impl SharedScrollInputState { queue_state.last_recorded = Some(event); - tracing::info!( + tracing::debug!( op = "scroll_input.queued", seq, delta_y = event.delta_y, @@ -131,6 +133,7 @@ impl SharedScrollInputState { Ok(waker_slot) => waker_slot.clone(), Err(poisoned) => poisoned.into_inner().clone(), }; + if let Some(event_waker) = event_waker { event_waker(); } @@ -151,6 +154,7 @@ impl SharedScrollInputState { while queue_state.queue.front().is_some_and(|event| event.seq <= after_seq) { let _ = queue_state.queue.pop_front(); + pruned_events = pruned_events.saturating_add(1); } @@ -168,7 +172,7 @@ impl SharedScrollInputState { if !replay.is_empty() || future_events > 0 || pruned_events > 0 { let newest_seq = queue_state.queue.back().map(|event| event.seq).unwrap_or(0); - tracing::info!( + tracing::debug!( op = "scroll_input.replay_window", after_seq, pruned_events, @@ -184,24 +188,6 @@ impl SharedScrollInputState { } } -type SharedScrollInputEventWaker = Arc; - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub(in crate::app) enum ScrollInputObserverStatus { - #[default] - Idle, - Starting, - Ready, - Failed, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(in crate::app) enum ScrollInputObserverWaitOutcome { - Ready, - TimedOut, - Failed, -} - #[derive(Default)] pub(in crate::app) struct ScrollInputObserverLifecycle { status: Mutex, @@ -323,6 +309,22 @@ struct SharedScrollInputQueueState { last_recorded: Option, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(in crate::app) enum ScrollInputObserverStatus { + #[default] + Idle, + Starting, + Ready, + Failed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::app) enum ScrollInputObserverWaitOutcome { + Ready, + TimedOut, + Failed, +} + #[cfg(test)] mod tests { use std::sync::{ @@ -437,7 +439,6 @@ mod tests { let _ = state.replay_after_seq_through(0, start + Duration::from_millis(2)); let _ = state.replay_after_seq_through(2, start + Duration::from_millis(2)); - let queue_state = state.queue_state.lock().unwrap(); let queued_seqs = queue_state.queue.iter().map(|event| event.seq).collect::>(); diff --git a/packages/rsnap-overlay/examples/scroll_capture_replay.rs b/packages/rsnap-overlay/examples/scroll_capture_replay.rs index 63830b37..f9d06ae1 100644 --- a/packages/rsnap-overlay/examples/scroll_capture_replay.rs +++ b/packages/rsnap-overlay/examples/scroll_capture_replay.rs @@ -2,15 +2,18 @@ #![allow(unused_crate_dependencies)] +use std::env; +use std::fs; use std::path::PathBuf; use std::process::ExitCode; -use color_eyre::eyre::WrapErr; +use color_eyre::Result; +use color_eyre::eyre::{self, WrapErr}; use directories::ProjectDirs; -use rsnap_overlay::replay_support::{ - RecordedScrollCaptureReplayMode, replay_recorded_scroll_capture_trace_with_mode, -}; -use tracing_subscriber::{EnvFilter, fmt}; +use tracing_subscriber::{self, EnvFilter}; + +use rsnap_overlay::replay_support::RecordedScrollCaptureReplaySummary; +use rsnap_overlay::replay_support::{self, RecordedScrollCaptureReplayMode}; fn main() -> ExitCode { if let Err(err) = run() { @@ -22,9 +25,10 @@ fn main() -> ExitCode { ExitCode::SUCCESS } -fn run() -> color_eyre::Result<()> { +fn run() -> Result<()> { color_eyre::install()?; - let _ = fmt() + + let _ = tracing_subscriber::fmt() .with_env_filter( EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new("warn,rsnap_overlay=info")), @@ -32,8 +36,7 @@ fn run() -> color_eyre::Result<()> { .with_target(false) .with_level(true) .try_init(); - - let mut args = std::env::args().skip(1); + let mut args = env::args().skip(1); let mut trace_manifest_path = None; let mut list_only = false; let mut emit_json = false; @@ -58,6 +61,7 @@ fn run() -> color_eyre::Result<()> { let Some(value) = args.next() else { color_eyre::eyre::bail!("--trace requires a manifest path"); }; + trace_manifest_path = Some(value); }, other => { @@ -82,8 +86,11 @@ fn run() -> color_eyre::Result<()> { } else { RecordedScrollCaptureReplayMode::RecordedSource }; - let mut summary = - replay_recorded_scroll_capture_trace_with_mode(&trace_manifest_path, replay_mode)?; + let mut summary = replay_support::replay_recorded_scroll_capture_trace_with_mode( + &trace_manifest_path, + replay_mode, + )?; + if summary_only { summary.step_results.clear(); } @@ -96,9 +103,7 @@ fn run() -> color_eyre::Result<()> { Ok(()) } -fn print_recorded_trace_summary( - summary: &rsnap_overlay::replay_support::RecordedScrollCaptureReplaySummary, -) { +fn print_recorded_trace_summary(summary: &RecordedScrollCaptureReplaySummary) { println!( "[replay] mode={:?} trace={} manifest={} final_export_height={} final_preview_height={} final_viewport_top_y={} recorded_final_export_height={:?} recorded_final_preview_height={:?} first_outcome_divergence_frame={:?} first_export_height_drift_frame={:?} first_preview_height_drift_frame={:?} first_semantic_issue_frame={:?} first_missed_downward_motion_frame={:?} first_underconsumed_downward_motion_frame={:?} first_growth_overshoot_frame={:?} max_recorded_committed_growth_rows={} max_replayed_committed_growth_rows={} max_recorded_export_jump={} max_recorded_preview_jump={} max_replayed_export_jump={} max_replayed_preview_jump={} final_preview_path={:?} final_export_path={:?}", summary.replay_mode, @@ -160,11 +165,11 @@ fn print_recorded_trace_summary( } } -fn latest_recorded_trace_manifest() -> color_eyre::Result { +fn latest_recorded_trace_manifest() -> Result { let project_dirs = ProjectDirs::from("ink", "hack", "rsnap") .expect("rsnap project directories should be available"); let trace_root = project_dirs.data_local_dir().join("scroll-capture-traces"); - let mut manifests: Vec = std::fs::read_dir(&trace_root) + let mut manifests: Vec = fs::read_dir(&trace_root) .wrap_err_with(|| format!("failed to read {}", trace_root.display()))? .filter_map(Result::ok) .map(|entry| entry.path().join("manifest.json")) @@ -172,8 +177,9 @@ fn latest_recorded_trace_manifest() -> color_eyre::Result { .collect(); manifests.sort(); + manifests.pop().ok_or_else(|| { - color_eyre::eyre::eyre!( + eyre::eyre!( "no recorded scroll-capture trace manifests found under {}; record a fresh live trace first or pass --trace ", trace_root.display() ) diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index c14de813..04b27bb8 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -961,6 +961,7 @@ fn rgba_image_from_cg_image(cg_image: &CGImage) -> Result { let data = CGDataProvider::data(Some(data_provider.as_ref())) .ok_or_else(|| eyre::eyre!("Failed to copy CGImage bytes"))?; let bytes_per_row = CGImage::bytes_per_row(Some(cg_image)); + rgba_image_from_bgra_rows(width, height, bytes_per_row, &data.to_vec()) } @@ -992,6 +993,7 @@ fn rgba_image_from_bgra_rows( } let mut buffer = Vec::with_capacity(width * height * 4); + for row in data[..required_len].chunks_exact(bytes_per_row) { buffer.extend_from_slice(&row[..expected_row_bytes]); } @@ -1435,7 +1437,7 @@ fn xcap_find_monitor(monitor: MonitorRect) -> Result { #[cfg(test)] mod tests { - use crate::backend::{CaptureBackend, StubCaptureBackend}; + use crate::backend::{self, CaptureBackend, StubCaptureBackend}; #[cfg(target_os = "macos")] use crate::state::{GlobalPoint, MonitorRect, RectPoints}; @@ -1483,8 +1485,7 @@ mod tests { 130, 140, 150, 255, 160, 170, 180, 255, // extra row 2 190, 200, 210, 255, 220, 230, 240, 255, // extra row 3 ]; - - let image = crate::backend::rgba_image_from_bgra_rows(width, height, bytes_per_row, &data) + let image = backend::rgba_image_from_bgra_rows(width, height, bytes_per_row, &data) .expect("image should decode"); assert_eq!(image.dimensions(), (2, 2)); @@ -1496,7 +1497,7 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn rgba_image_from_bgra_rows_rejects_short_backing_store() { - let err = crate::backend::rgba_image_from_bgra_rows( + let err = backend::rgba_image_from_bgra_rows( 2, 2, 8, diff --git a/packages/rsnap-overlay/src/lib.rs b/packages/rsnap-overlay/src/lib.rs index 1a8b2d38..93efef2e 100644 --- a/packages/rsnap-overlay/src/lib.rs +++ b/packages/rsnap-overlay/src/lib.rs @@ -12,7 +12,6 @@ pub mod bench_support { ScrollCaptureOverlapMetrics, ScrollCaptureSessionMetrics, }; } - /// Deterministic replay harness exports for scroll-capture verification. pub mod replay_support { pub use crate::overlay::replay_support::{ diff --git a/packages/rsnap-overlay/src/live_frame_stream_macos.rs b/packages/rsnap-overlay/src/live_frame_stream_macos.rs index b302b45f..14b817cb 100644 --- a/packages/rsnap-overlay/src/live_frame_stream_macos.rs +++ b/packages/rsnap-overlay/src/live_frame_stream_macos.rs @@ -124,58 +124,6 @@ const STREAM_ERROR_TIMEOUT_CODE: isize = 1; const STREAM_ERROR_NULL_CONTENT_CODE: isize = 2; const STREAM_ERROR_RETAIN_FAILED_CODE: isize = 3; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct StreamCaptureRegion { - rect_points: RectPoints, - rect_pixels: RectPoints, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum StreamCaptureTarget { - FullMonitor, - Region(StreamCaptureRegion), -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum StreamFilterMode { - ExcludeCurrentProcess, - ExcludeCurrentProcessShareableWindows, -} - -enum WorkerRequest { - EnsureMonitor { - monitor: MonitorRect, - }, - RefreshMonitor { - monitor: MonitorRect, - }, - SampleCursor { - monitor: MonitorRect, - x_px: u32, - y_px: u32, - want_patch: bool, - patch_width_px: u32, - patch_height_px: u32, - reply_tx: Sender>, - }, - LatestRgbaSnapshot { - monitor: MonitorRect, - reply_tx: Sender>>, - }, - LatestRgbaRegion { - monitor: MonitorRect, - rect_px: RectPoints, - reply_tx: Sender>, - }, - OrderedRgbaRegionsAfterSeq { - monitor: MonitorRect, - rect_px: RectPoints, - after_frame_seq: u64, - reply_tx: Sender>>, - }, - Shutdown, -} - pub(crate) struct OrderedRegionFrame { pub(crate) frame_seq: u64, pub(crate) captured_at: Instant, @@ -330,6 +278,7 @@ impl MacLiveFrameStream { }, Err(poisoned) => { let mut guard = poisoned.into_inner(); + *guard = Some(kind); }, } @@ -476,6 +425,7 @@ impl MacLiveFrameStream { let frames = self.shared_latest_frame.frames_after_seq_for_monitor(monitor.id, after_frame_seq); + if frames.is_empty() { if self.shared_latest_frame.latest_frame_for_monitor(monitor.id).is_none() { self.prime_monitor_nonblocking(monitor); @@ -486,6 +436,7 @@ impl MacLiveFrameStream { let stream_rect_px = self.stream_rect_for_requested_region(rect_px)?; let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); + (!frames.is_empty()).then_some(frames) } @@ -527,6 +478,7 @@ impl MacLiveFrameStream { if self.shared_latest_frame.waiting_for_frame_after_setup(monitor.id) { return false; } + let Some(latest_frame) = self.shared_latest_frame.latest_frame_for_monitor(monitor.id) else { self.prime_monitor_nonblocking(monitor); @@ -556,52 +508,6 @@ impl MacLiveFrameStream { } } -fn stream_rect_for_requested_region( - capture_target: StreamCaptureTarget, - requested_rect_px: RectPoints, -) -> Option { - match capture_target { - StreamCaptureTarget::FullMonitor => Some(requested_rect_px), - StreamCaptureTarget::Region(region) => { - let relative_x = requested_rect_px.x.checked_sub(region.rect_pixels.x)?; - let relative_y = requested_rect_px.y.checked_sub(region.rect_pixels.y)?; - let requested_right = requested_rect_px.x.checked_add(requested_rect_px.width)?; - let requested_bottom = requested_rect_px.y.checked_add(requested_rect_px.height)?; - let region_right = region.rect_pixels.x.checked_add(region.rect_pixels.width)?; - let region_bottom = region.rect_pixels.y.checked_add(region.rect_pixels.height)?; - - if requested_right > region_right || requested_bottom > region_bottom { - return None; - } - - Some(RectPoints::new( - relative_x, - relative_y, - requested_rect_px.width, - requested_rect_px.height, - )) - }, - } -} - -fn should_refresh_monitor_frame( - latest_frame_seq: u64, - after_frame_seq: u64, - frame_age: Duration, - force_refresh: bool, -) -> bool { - if latest_frame_seq > after_frame_seq { - return false; - } - - if force_refresh { - let _ = frame_age; - return true; - } - - frame_age > STREAM_REGION_FRAME_MAX_AGE -} - impl Drop for MacLiveFrameStream { fn drop(&mut self) { let _ = self.request_tx.send(WorkerRequest::Shutdown); @@ -612,6 +518,12 @@ impl Drop for MacLiveFrameStream { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct StreamCaptureRegion { + rect_points: RectPoints, + rect_pixels: RectPoints, +} + #[derive(Clone, Debug, Default)] struct StreamFilterConfig { self_capture_exception_window_ids: Vec, @@ -654,25 +566,6 @@ struct SharedLatestFrame { pending_refresh_monitor: Mutex>, waiting_for_frame_until: Mutex>, } - -#[derive(Clone, Copy)] -struct PendingMonitorRequest { - monitor_id: u32, - stalled_after_frame_seq: u64, - started_at: Instant, -} - -struct StoreFrameOutcome { - completed_ensure: bool, - completed_refresh: bool, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum StreamRequestProgress { - AwaitingFirstFrame, - Settled, -} - impl SharedLatestFrame { fn store(&self, monitor_id: u32, frame: &QueuedPixelBufferFrame) -> StoreFrameOutcome { match self.frames.lock() { @@ -681,13 +574,16 @@ impl SharedLatestFrame { monitor_id, frames: VecDeque::with_capacity(STREAM_FRAME_QUEUE_CAPACITY), }); + if shared.monitor_id != monitor_id { shared.monitor_id = monitor_id; + shared.frames.clear(); } if shared.frames.len() >= STREAM_FRAME_QUEUE_CAPACITY { shared.frames.pop_front(); } + shared.frames.push_back(frame.clone()); }, Err(poisoned) => { @@ -696,13 +592,16 @@ impl SharedLatestFrame { monitor_id, frames: VecDeque::with_capacity(STREAM_FRAME_QUEUE_CAPACITY), }); + if shared.monitor_id != monitor_id { shared.monitor_id = monitor_id; + shared.frames.clear(); } if shared.frames.len() >= STREAM_FRAME_QUEUE_CAPACITY { shared.frames.pop_front(); } + shared.frames.push_back(frame.clone()); }, } @@ -712,7 +611,6 @@ impl SharedLatestFrame { fn complete_pending_requests_for_stored_frame(&self, monitor_id: u32) -> StoreFrameOutcome { self.clear_waiting_for_frame(monitor_id); - StoreFrameOutcome { completed_ensure: self.finish_ensure_monitor(monitor_id), completed_refresh: self.finish_refresh_monitor(monitor_id), @@ -843,6 +741,7 @@ impl SharedLatestFrame { { return false; } + tracing::info!( op = "live_frame_stream.stale_pending_refresh_recovered", monitor_id, @@ -888,6 +787,7 @@ impl SharedLatestFrame { { return false; } + tracing::info!( op = "live_frame_stream.stale_pending_refresh_recovered", monitor_id, @@ -947,12 +847,14 @@ impl SharedLatestFrame { let Some((pending_monitor_id, until)) = *guard else { return false; }; + if pending_monitor_id != monitor_id { return false; } if now < until { return true; } + *guard = None; }, Err(poisoned) => { @@ -960,12 +862,14 @@ impl SharedLatestFrame { let Some((pending_monitor_id, until)) = *guard else { return false; }; + if pending_monitor_id != monitor_id { return false; } if now < until { return true; } + *guard = None; }, } @@ -1014,6 +918,18 @@ impl SharedLatestFrame { } } +#[derive(Clone, Copy)] +struct PendingMonitorRequest { + monitor_id: u32, + stalled_after_frame_seq: u64, + started_at: Instant, +} + +struct StoreFrameOutcome { + completed_ensure: bool, + completed_refresh: bool, +} + struct StreamOutputIvars { monitor_id: u32, frames: Mutex>, @@ -1057,6 +973,69 @@ impl CurrentProcessExceptionWindows { } } +struct RefreshStreamArgs<'a> { + state: &'a mut Option, + last_setup_attempt_at: &'a mut Option, + monitor: MonitorRect, + filter: &'a StreamFilterConfig, + capture_target: StreamCaptureTarget, + frame_waker: Option>, + frame_seq_counter: Arc, + shared_latest_frame: Arc, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamCaptureTarget { + FullMonitor, + Region(StreamCaptureRegion), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamFilterMode { + ExcludeCurrentProcess, + ExcludeCurrentProcessShareableWindows, +} + +enum WorkerRequest { + EnsureMonitor { + monitor: MonitorRect, + }, + RefreshMonitor { + monitor: MonitorRect, + }, + SampleCursor { + monitor: MonitorRect, + x_px: u32, + y_px: u32, + want_patch: bool, + patch_width_px: u32, + patch_height_px: u32, + reply_tx: Sender>, + }, + LatestRgbaSnapshot { + monitor: MonitorRect, + reply_tx: Sender>>, + }, + LatestRgbaRegion { + monitor: MonitorRect, + rect_px: RectPoints, + reply_tx: Sender>, + }, + OrderedRgbaRegionsAfterSeq { + monitor: MonitorRect, + rect_px: RectPoints, + after_frame_seq: u64, + reply_tx: Sender>>, + }, + Shutdown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamRequestProgress { + AwaitingFirstFrame, + Settled, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum StreamReuseDecision { SetupFresh, @@ -1107,6 +1086,52 @@ impl StreamOutput { } } +fn stream_rect_for_requested_region( + capture_target: StreamCaptureTarget, + requested_rect_px: RectPoints, +) -> Option { + match capture_target { + StreamCaptureTarget::FullMonitor => Some(requested_rect_px), + StreamCaptureTarget::Region(region) => { + let relative_x = requested_rect_px.x.checked_sub(region.rect_pixels.x)?; + let relative_y = requested_rect_px.y.checked_sub(region.rect_pixels.y)?; + let requested_right = requested_rect_px.x.checked_add(requested_rect_px.width)?; + let requested_bottom = requested_rect_px.y.checked_add(requested_rect_px.height)?; + let region_right = region.rect_pixels.x.checked_add(region.rect_pixels.width)?; + let region_bottom = region.rect_pixels.y.checked_add(region.rect_pixels.height)?; + + if requested_right > region_right || requested_bottom > region_bottom { + return None; + } + + Some(RectPoints::new( + relative_x, + relative_y, + requested_rect_px.width, + requested_rect_px.height, + )) + }, + } +} + +fn should_refresh_monitor_frame( + latest_frame_seq: u64, + after_frame_seq: u64, + frame_age: Duration, + force_refresh: bool, +) -> bool { + if latest_frame_seq > after_frame_seq { + return false; + } + if force_refresh { + let _ = frame_age; + + return true; + } + + frame_age > STREAM_REGION_FRAME_MAX_AGE +} + fn stream_worker_loop( request_rx: Receiver, frame_waker: Option>, @@ -1193,6 +1218,7 @@ fn handle_stream_worker_request( patch_height_px, reply_tx, ); + true }, WorkerRequest::LatestRgbaSnapshot { monitor, reply_tx } => { @@ -1207,6 +1233,7 @@ fn handle_stream_worker_request( shared_latest_frame, reply_tx, ); + true }, WorkerRequest::LatestRgbaRegion { monitor, rect_px, reply_tx } => { @@ -1222,6 +1249,7 @@ fn handle_stream_worker_request( shared_latest_frame, reply_tx, ); + true }, WorkerRequest::OrderedRgbaRegionsAfterSeq { @@ -1244,6 +1272,7 @@ fn handle_stream_worker_request( reply_tx, false, ); + true }, WorkerRequest::Shutdown => false, @@ -1267,6 +1296,7 @@ fn handle_ensure_monitor_request( current_monitor_id = state.as_ref().map(|current| current.monitor_id), "Handling an asynchronous ScreenCaptureKit ensure request." ); + let progress = ensure_stream( state, last_setup_attempt_at, @@ -1278,6 +1308,7 @@ fn handle_ensure_monitor_request( frame_seq_counter, shared_latest_frame.clone(), ); + if progress == StreamRequestProgress::Settled { shared_latest_frame.finish_ensure_monitor(monitor.id); } @@ -1314,6 +1345,7 @@ fn handle_refresh_monitor_request( current_monitor_id = state.as_ref().map(|current| current.monitor_id), "Handling an asynchronous ScreenCaptureKit refresh request." ); + let progress = refresh_stream_nonblocking( state, last_setup_attempt_at, @@ -1324,6 +1356,7 @@ fn handle_refresh_monitor_request( frame_seq_counter, shared_latest_frame.clone(), ); + if progress == StreamRequestProgress::Settled { shared_latest_frame.finish_refresh_monitor(monitor.id); } @@ -1506,6 +1539,7 @@ fn refresh_stream_nonblocking( backoff_ms = STREAM_SETUP_BACKOFF.as_millis(), "Skipped ScreenCaptureKit refresh because setup backoff is still active." ); + return StreamRequestProgress::Settled; } @@ -1516,6 +1550,7 @@ fn refresh_stream_nonblocking( current_monitor_id = state.as_ref().map(|current| current.monitor_id), "Refresh request found no matching live stream and is falling back to ensure." ); + return ensure_stream( state, last_setup_attempt_at, @@ -1577,6 +1612,7 @@ fn ensure_stream( backoff_ms = setup_backoff.as_millis(), "Skipped ScreenCaptureKit setup because the current setup backoff window is still active." ); + return StreamRequestProgress::Settled; } @@ -1597,6 +1633,7 @@ fn ensure_stream( had_existing_state = state.is_some(), "ScreenCaptureKit setup did not produce a usable live stream." ); + return StreamRequestProgress::Settled; }; @@ -1607,6 +1644,7 @@ fn ensure_stream( monitor_id = monitor.id, "Retained the current live stream because the replacement setup still lacked complete self-capture exception windows." ); + let mut next_state = Some(next_state); teardown_stream(&mut next_state); @@ -1617,7 +1655,9 @@ fn ensure_stream( let mut previous_state = state.replace(next_state); teardown_stream(&mut previous_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( op = "live_frame_stream.ensure_stream_ready", monitor_id = monitor.id, @@ -1633,8 +1673,11 @@ fn ensure_stream( teardown_stream(state); let exceptions_complete = next_state.self_capture_exception_window_ids_complete; + *state = Some(next_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( op = "live_frame_stream.ensure_stream_ready", monitor_id = monitor.id, @@ -1671,7 +1714,6 @@ fn latest_fresh_rgba_region( frame_seq_counter.clone(), shared_latest_frame.clone(), ); - let now = Instant::now(); let stream_state = state.as_ref()?; @@ -1691,7 +1733,6 @@ fn latest_fresh_rgba_region( frame_seq_counter, shared_latest_frame, }); - let min_captured_at = Instant::now(); let deadline = min_captured_at + STREAM_REGION_FRAME_REFRESH_TIMEOUT; @@ -1737,7 +1778,6 @@ fn ordered_queued_rgba_regions_after_seq_nonblocking( frame_seq_counter, shared_latest_frame, ); - let stream_state = state.as_ref()?; let frames = stream_state.output.queued_frames_after_seq(after_frame_seq); let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); @@ -1770,7 +1810,6 @@ fn ordered_fresh_rgba_regions_after_seq( frame_seq_counter.clone(), shared_latest_frame.clone(), ); - let stream_state = state.as_ref()?; let frames = stream_state.output.queued_frames_after_seq(after_frame_seq); let frames = ordered_rgba_regions_from_frames(frames, stream_rect_px); @@ -1797,7 +1836,6 @@ fn ordered_fresh_rgba_regions_after_seq( frame_seq_counter, shared_latest_frame, }); - let min_captured_at = Instant::now(); let deadline = min_captured_at + STREAM_REGION_FRAME_REFRESH_TIMEOUT; @@ -1828,12 +1866,14 @@ fn refresh_stream(args: RefreshStreamArgs<'_>) -> StreamRequestProgress { frame_seq_counter, shared_latest_frame, } = args; + tracing::info!( op = "live_frame_stream.refresh_stream_begin", monitor_id = monitor.id, current_monitor_id = state.as_ref().map(|current| current.monitor_id), "Refreshing the ScreenCaptureKit live stream." ); + *last_setup_attempt_at = Some(Instant::now()); let Some(next_state) = setup_stream_for_monitor( @@ -1851,7 +1891,9 @@ fn refresh_stream(args: RefreshStreamArgs<'_>) -> StreamRequestProgress { let mut previous_state = state.replace(next_state); teardown_stream(&mut previous_state); + shared_latest_frame.mark_waiting_for_frame(monitor.id); + tracing::info!( op = "live_frame_stream.refresh_stream_ready", monitor_id = monitor.id, @@ -1863,26 +1905,17 @@ fn refresh_stream(args: RefreshStreamArgs<'_>) -> StreamRequestProgress { StreamRequestProgress::AwaitingFirstFrame } -struct RefreshStreamArgs<'a> { - state: &'a mut Option, - last_setup_attempt_at: &'a mut Option, - monitor: MonitorRect, - filter: &'a StreamFilterConfig, - capture_target: StreamCaptureTarget, - frame_waker: Option>, - frame_seq_counter: Arc, - shared_latest_frame: Arc, -} - fn teardown_stream(state: &mut Option) { let Some(state) = state.take() else { return; }; + tracing::info!( op = "live_frame_stream.teardown_stream", monitor_id = state.monitor_id, "Stopping the current ScreenCaptureKit live stream." ); + let stop_block = RcBlock::new(|_err: *mut NSError| {}); unsafe { state.stream.stopCaptureWithCompletionHandler(Some(&stop_block)) }; @@ -1983,12 +2016,16 @@ fn setup_stream_for_monitor( .is_err() { log_add_stream_output_failed(monitor.id, filter_mode); + return None; } + if let Err(error) = start_capture_blocking(&stream) { log_start_capture_failed(monitor.id, filter_mode, &error); + return None; } + tracing::info!( op = "live_frame_stream.setup_stream_ready", monitor_id = monitor.id, @@ -2563,9 +2600,9 @@ fn ordered_rgba_regions_from_frames( #[cfg(test)] mod tests { + use std::ptr::{self, NonNull}; use std::sync::{Arc, atomic::AtomicU64}; use std::time::Duration; - use std::{ptr, ptr::NonNull}; use objc2_core_foundation::CFRetained; use objc2_core_video::{CVPixelBufferCreate, kCVPixelFormatType_32BGRA, kCVReturnSuccess}; @@ -2585,6 +2622,7 @@ mod tests { NonNull::from(&mut buffer), ) }; + assert_eq!(res, kCVReturnSuccess); live_frame_stream_macos::SharedPixelBuffer(unsafe { @@ -2680,6 +2718,7 @@ mod tests { let now = std::time::Instant::now(); assert!(shared.begin_ensure_monitor(7)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); let outcome = shared.complete_pending_requests_for_stored_frame(7); @@ -2696,6 +2735,7 @@ mod tests { let now = std::time::Instant::now(); assert!(shared.begin_refresh_monitor(7, 11, now)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); let outcome = shared.complete_pending_requests_for_stored_frame(9); @@ -2712,6 +2752,7 @@ mod tests { let now = std::time::Instant::now(); assert!(shared.begin_refresh_monitor(7, 11, now)); + shared.mark_waiting_for_frame_until(7, now + Duration::from_secs(1)); let outcome = shared.complete_pending_requests_for_stored_frame(7); @@ -2771,7 +2812,7 @@ mod tests { let monitor = crate::state::MonitorRect { id: 7, origin: crate::state::GlobalPoint::new(0, 0), - width: 1440, + width: 1_440, height: 900, scale_factor_x1000: 2_000, }; @@ -2779,6 +2820,7 @@ mod tests { let mut last_setup_attempt_at = None; assert!(shared.begin_refresh_monitor(monitor.id, 11, now)); + shared.mark_waiting_for_frame_until(monitor.id, now + Duration::from_secs(1)); assert!(live_frame_stream_macos::handle_refresh_monitor_request( @@ -2823,7 +2865,7 @@ mod tests { let monitor = crate::state::MonitorRect { id: 7, origin: crate::state::GlobalPoint::new(0, 0), - width: 1440, + width: 1_440, height: 900, scale_factor_x1000: 2_000, }; @@ -2833,7 +2875,6 @@ mod tests { captured_at: std::time::Instant::now(), pixel_buffer: test_pixel_buffer(), }; - let _ = stream.shared_latest_frame.store(monitor.id, &frame); assert!(stream.ordered_rgba_regions_after_seq_nonblocking(monitor, rect, 4).is_none()); @@ -2878,7 +2919,7 @@ mod tests { let monitor = crate::state::MonitorRect { id: 7, origin: crate::state::GlobalPoint::new(0, 0), - width: 1440, + width: 1_440, height: 900, scale_factor_x1000: 2_000, }; @@ -2898,7 +2939,7 @@ mod tests { let monitor = crate::state::MonitorRect { id: 7, origin: crate::state::GlobalPoint::new(0, 0), - width: 1440, + width: 1_440, height: 900, scale_factor_x1000: 2_000, }; diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 9e189a83..e712051f 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -1,12 +1,14 @@ +pub(crate) mod replay_support; + mod hud_helpers; mod image_helpers; mod output; -pub(crate) mod replay_support; mod scroll_runtime; mod session_state; mod trace_recording; mod window_runtime; +use std::collections::VecDeque; #[cfg(target_os = "macos")] use std::ffi::c_void; use std::mem; @@ -27,7 +29,6 @@ use std::{ use color_eyre::eyre::{self, Result, WrapErr}; #[cfg(not(target_os = "macos"))] use device_query::{DeviceQuery, Keycode}; -use egui::ColorImage; use egui::FullOutput; use egui::Mesh; use egui::Painter; @@ -36,9 +37,10 @@ use egui::TextureId; use egui::TextureOptions; use egui::Ui; use egui::{ - self, Align, Align2, Color32, CornerRadius, Event, FontDefinitions, FontFamily, FontId, Frame, - Layout, Margin, PointerButton, Pos2, Rect, Vec2, + self, Align2, Color32, CornerRadius, Event, FontDefinitions, FontFamily, FontId, Frame, Layout, + Margin, PointerButton, Pos2, Rect, Vec2, }; +use egui::{Align, ColorImage}; use egui::{ Area, CentralPanel, ClippedPrimitive, Id, LayerId, Order, RichText, Sense, Shape, Stroke, StrokeKind, UiBuilder, ViewportId, Visuals, @@ -137,7 +139,7 @@ use self::trace_recording::{ use crate::backend; #[cfg(target_os = "macos")] use crate::live_frame_stream_macos::{CursorSampleRequest, MacLiveFrameStream}; -use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; +use crate::scroll_capture::{self, ScrollDirection, ScrollObserveOutcome, ScrollSession}; use crate::state::LiveCursorSample; use crate::worker::CapturedMonitorRegionResult; use crate::{ @@ -303,8 +305,7 @@ const SCROLL_PREVIEW_WINDOW_WIDTH_POINTS: f64 = 260.0; const SCROLL_PREVIEW_WINDOW_HEIGHT_POINTS: f64 = 360.0; const SCROLL_PREVIEW_WINDOW_MARGIN_POINTS: i32 = 16; const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); -const SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL: Duration = - Duration::from_millis(60); +const SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL: Duration = Duration::from_millis(60); #[cfg(target_os = "macos")] const SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE: Duration = Duration::from_millis(180); const SCROLL_CAPTURE_PREVIEW_WIDTH_PX: u32 = 320; @@ -1988,9 +1989,11 @@ impl OverlaySession { let consumed_live_stream_stale_grace = false; let allow_gate_bypass = allow_stale_input || consumed_live_stream_stale_grace; let motion_rows_hint = self.scroll_capture_commit_motion_rows_hint_at(observation_at); + if !allow_gate_bypass && prior_block_reason.is_some() { return Some(Ok(ScrollObserveOutcome::NoChange)); } + let result = { let Some(session) = self.scroll_capture.session.as_mut() else { self.scroll_capture_set_error("Scroll capture session is unavailable."); @@ -2004,6 +2007,7 @@ impl OverlaySession { allow_post_stall_burst_search, ) }; + if let Ok(outcome) = &result { self.consume_scroll_capture_downward_motion_rows_for_outcome(outcome); } @@ -2016,9 +2020,8 @@ impl OverlaySession { return None; } - let Some(input_direction_at) = self.scroll_capture.input_direction_at else { - return None; - }; + let input_direction_at = self.scroll_capture.input_direction_at?; + if !self.scroll_capture.input_gesture_active && observation_at.saturating_duration_since(input_direction_at) > SCROLL_CAPTURE_INPUT_FRESHNESS @@ -2036,6 +2039,7 @@ impl OverlaySession { fn sync_scroll_preview_segments(&mut self) { let image = self.current_scroll_preview_render_image(); + { let Some(preview) = self.scroll_preview_window.as_mut() else { return; @@ -2057,6 +2061,7 @@ impl OverlaySession { fn refresh_scroll_preview_display_image(&mut self) { let motion_rows_hint = None; + self.scroll_capture.last_overlay_preview_motion_rows_hint = motion_rows_hint; self.scroll_capture.last_overlay_preview_provisional_motion_rows_hint = None; self.scroll_capture.last_overlay_preview_existing_candidate_height = None; @@ -2071,6 +2076,7 @@ impl OverlaySession { self.scroll_capture.last_overlay_preview_latest_frame_present = self.scroll_capture.preview_latest_frame.is_some(); self.scroll_capture.last_overlay_preview_used_provisional = false; + if let Some(session) = self.scroll_capture.session.as_mut() { self.scroll_capture.preview_committed_image = Some(session.export_image().clone()); self.scroll_capture.preview_display_image = @@ -2081,7 +2087,7 @@ impl OverlaySession { self.scroll_capture.preview_display_image = self.scroll_capture.preview_committed_image.as_ref().map(|base_preview| { - crate::scroll_capture::compose_provisional_preview_image( + scroll_capture::compose_provisional_preview_image( base_preview, self.scroll_capture.preview_latest_frame.as_ref(), motion_rows_hint, @@ -2098,6 +2104,7 @@ impl OverlaySession { fn scroll_preview_display_size_points(&self) -> Option { let [width_px, height_px] = self.scroll_capture_preview_dimensions()?; + if width_px == 0 || height_px == 0 { return None; } @@ -5216,6 +5223,7 @@ impl OverlaySession { } let remaining = self.scroll_capture.downward_motion_rows_pending - f64::from(consumed_rows); + self.scroll_capture.downward_motion_rows_pending = remaining.max(0.0); } @@ -5352,6 +5360,7 @@ impl OverlaySession { #[cfg(target_os = "macos")] let _cursor_inside_capture_rect = capture_rect.contains(cursor_pixels); + #[cfg(target_os = "macos")] if delta_y != 0.0 && !gesture_ended @@ -5436,7 +5445,6 @@ impl OverlaySession { if !self.scroll_capture_has_fresh_downward_backlog_at(now) { return false; } - if self.scroll_capture.input_gesture_active { return false; } @@ -5847,7 +5855,7 @@ impl OverlaySession { self.scroll_frame_waker.clone(), )), #[cfg(target_os = "macos")] - live_stream_backlog: std::collections::VecDeque::new(), + live_stream_backlog: VecDeque::new(), #[cfg(target_os = "macos")] last_stream_frame_seq: 0, #[cfg(target_os = "macos")] @@ -5952,6 +5960,7 @@ impl OverlaySession { } let base_frame_dimensions = base_frame.dimensions(); + self.scroll_capture = match self.build_scroll_capture_state( monitor, capture_rect_points, @@ -5970,7 +5979,6 @@ impl OverlaySession { if let Some(hook) = self.scroll_capture_started_hook.clone() { hook(); } - if let Some(trace_recorder) = self.scroll_capture.trace_recorder.as_ref() { tracing::info!( op = "scroll_capture.trace_recording_enabled", @@ -6005,7 +6013,6 @@ impl OverlaySession { preview.window.set_visible(true); preview.window.request_redraw(); } - if let (Some(monitor), Some(live_stream)) = (self.scroll_capture.monitor, self.scroll_capture.live_stream.as_ref()) { @@ -6062,8 +6069,8 @@ impl OverlaySession { if !session.undo_last_append() { return; } - self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_committed_image(); self.clear_scroll_capture_inflight_request(); #[cfg(target_os = "macos")] @@ -6094,7 +6101,6 @@ impl OverlaySession { return; } - if self.scroll_capture.active { self.maybe_tick_scroll_capture(); self.refresh_scroll_preview_committed_image(); @@ -6837,7 +6843,6 @@ impl OverlaySession { if self.scroll_capture.overlay_mouse_passthrough_persistent { return; } - if !self.scroll_capture.overlay_mouse_passthrough_active { return; } @@ -7255,6 +7260,7 @@ impl OverlaySession { OverlayExit::Saved(path) => ("saved", None, Some(path.display().to_string()), None), OverlayExit::Error(message) => ("error", None, None, Some(message.as_str())), }; + tracing::info!( op = "overlay.exit_begin", exit_kind, @@ -7270,25 +7276,30 @@ impl OverlaySession { last_event_detail = ?self.event_loop_last_progress_detail, "Beginning overlay exit cleanup." ); + if self.scroll_capture.active { self.maybe_tick_scroll_capture(); self.refresh_scroll_preview_committed_image(); self.refresh_scroll_preview_display_image(); self.sync_scroll_preview_segments(); } + let scroll_capture_final_snapshot = self.scroll_capture_trace_snapshot_at(Instant::now()); let final_preview_image = self.current_scroll_preview_render_image(); + if let (Some(trace_recorder), Some(session)) = (self.scroll_capture.trace_recorder.as_mut(), self.scroll_capture.session.as_ref()) { let final_preview_image = final_preview_image.unwrap_or_else(|| session.preview_image().clone()); + trace_recorder.finalize_session( session, &final_preview_image, scroll_capture_final_snapshot, ); } + #[cfg(target_os = "macos")] self.set_scroll_overlay_mouse_passthrough(false); self.windows.clear(); @@ -8009,6 +8020,7 @@ impl ScrollPreviewWindow { match self.preview_image.as_mut() { Some(strip) if strip.pixel_size == pixel_size => { strip.texture.set(color_image, TextureOptions::LINEAR); + strip.pixel_size = pixel_size; strip.rgba = rgba; strip.size_points = size_points; @@ -8057,7 +8069,7 @@ impl ScrollPreviewWindow { (available.x / preview_image.size_points.x).clamp(0.05, 1.0); let draw_size = preview_image.size_points * scale; - ui.with_layout(Layout::top_down(egui::Align::Center), |ui| { + ui.with_layout(Layout::top_down(Align::Center), |ui| { ui.image((preview_image.texture.id(), draw_size)); }); } else { @@ -12961,6 +12973,7 @@ mod tests { use std::collections::VecDeque; #[cfg(target_os = "macos")] use std::sync::Arc; + use std::thread; #[cfg(target_os = "macos")] use std::time::Duration; use std::time::Instant; @@ -13119,12 +13132,12 @@ mod tests { height: u32, start_row: u32, ) -> image::RgbaImage { - let mut image = make_sparse_worker_capture_window(width, height, start_row); let scrollbar_left = width.saturating_sub(18); let content_left = 56_u32; let content_right = width.saturating_sub(48); let heading_width = 220_u32; let paragraph_width = content_right.saturating_sub(content_left); + let mut image = make_sparse_worker_capture_window(width, height, start_row); for y in 0..height { let document_row = start_row.saturating_add(y); @@ -13136,19 +13149,19 @@ mod tests { } else if document_row % 420 >= 54 && document_row % 420 < 220 { if document_row % 24 < 3 { let trim = ((document_row / 24) % 5) * 18; + for x in content_left ..content_left.saturating_add(paragraph_width.saturating_sub(trim)) { image.put_pixel(x, y, Rgba([72, 72, 72, 255])); } } - } else if document_row % 420 >= 270 && document_row % 420 < 360 { - if document_row % 20 < 2 { - for x in content_left.saturating_add(20) - ..content_left.saturating_add(paragraph_width.saturating_sub(70)) - { - image.put_pixel(x, y, Rgba([98, 98, 98, 255])); - } + } else if document_row % 420 >= 270 && document_row % 420 < 360 && document_row % 20 < 2 + { + for x in content_left.saturating_add(20) + ..content_left.saturating_add(paragraph_width.saturating_sub(70)) + { + image.put_pixel(x, y, Rgba([98, 98, 98, 255])); } } @@ -13160,6 +13173,7 @@ mod tests { let thumb_height = (height / 5).max(16); let thumb_top = (start_row / 3) % height.max(thumb_height + 1); let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); + for y in thumb_top..thumb_top.saturating_add(thumb_height) { for x in scrollbar_left.saturating_add(3)..width.saturating_sub(4) { image.put_pixel(x, y, Rgba([96, 96, 96, 255])); @@ -13184,10 +13198,12 @@ mod tests { fn drain_scroll_capture_worker_until_idle(session: &mut OverlaySession) { for _ in 0..64 { let _ = session.drain_worker_responses(); + if session.scroll_capture.inflight_request_id.is_none() { return; } - std::thread::sleep(Duration::from_millis(5)); + + thread::sleep(Duration::from_millis(5)); } panic!( @@ -14855,9 +14871,9 @@ mod tests { let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); let mismatched_preview = RgbaImage::from_pixel(320, 40, Rgba([99, 0, 0, 255])); let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); - let _ = scroll_session.observe_downward_sample(grown).expect("observe"); let expected_export = scroll_session.export_image().clone(); + session.scroll_capture.active = true; session.scroll_capture.session = Some(scroll_session); session.scroll_capture.preview_display_image = Some(mismatched_preview.clone()); @@ -14867,8 +14883,8 @@ mod tests { #[test] fn current_scroll_preview_render_image_uses_preview_display_when_scroll_capture_is_inactive() { - let mut session = OverlaySession::new(); let preview = RgbaImage::from_pixel(320, 64, Rgba([42, 0, 0, 255])); + let mut session = OverlaySession::new(); session.scroll_capture.preview_display_image = Some(preview.clone()); @@ -14882,9 +14898,9 @@ mod tests { let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); let mismatched_preview = RgbaImage::from_pixel(320, 40, Rgba([99, 0, 0, 255])); let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); - let _ = scroll_session.observe_downward_sample(grown).expect("observe"); let expected_export = scroll_session.export_image().clone(); + session.scroll_capture.active = true; session.scroll_capture.session = Some(scroll_session); session.scroll_capture.preview_display_image = Some(mismatched_preview.clone()); @@ -14902,13 +14918,13 @@ mod tests { let base = make_scroll_capture_test_image(3, &[[10, 0, 0, 255]; 8]); let grown = make_scroll_capture_test_image(3, &[[20, 0, 0, 255]; 12]); let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); - let _ = scroll_session.observe_downward_sample(grown).expect("observe"); let expected_committed = scroll_session.export_image().clone(); let expected_render = scroll_session.export_image().clone(); session.scroll_capture.active = true; session.scroll_capture.session = Some(scroll_session); + session.refresh_scroll_preview_committed_image(); session.refresh_scroll_preview_display_image(); @@ -14952,11 +14968,11 @@ mod tests { let mut scroll_session = ScrollSession::new(base, 320).expect("scroll session"); let _ = scroll_session.observe_downward_sample(grown).expect("observe"); let expected_export = scroll_session.export_image().clone(); - let monitor = test_monitor(); session.state.begin_freeze(monitor); session.state.finish_freeze(monitor, test_frozen_image()); + session.state.frozen_capture_rect = Some(RectPoints::new(100, 120, 220, 180)); session.frozen_capture_source = FrozenCaptureSource::DragRegion; session.authoritative_frozen_capture_ready = true; @@ -15173,6 +15189,7 @@ mod tests { let mut session = OverlaySession::new(); seed_ready_scroll_capture_selection(&mut session); + session.live_sample_stream = Some(MacLiveFrameStream::new()); let control = session.start_scroll_capture(); @@ -16419,6 +16436,7 @@ mod tests { let mut session = OverlaySession::new(); seed_ready_scroll_capture_selection(&mut session); + let _ = session.start_scroll_capture(); session.toggle_scroll_capture_paused(); @@ -16785,9 +16803,9 @@ mod tests { scale_factor_x1000: 1_000, }; let capture_rect = RectPoints::new(100, 120, 200, 240); - let mut session = OverlaySession::new(); let base = make_sparse_worker_capture_window(512, 640, 0); let next = make_sparse_worker_capture_window(512, 640, 90); + let mut session = OverlaySession::new(); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(monitor); @@ -16828,9 +16846,9 @@ mod tests { fn stale_same_direction_worker_frame_keeps_latched_worker_observation_context() { let monitor = test_monitor(); let capture_rect = RectPoints::new(100, 120, 512, 640); - let mut session = OverlaySession::new(); let base = make_sparse_worker_capture_window(512, 640, 0); let next = make_sparse_worker_capture_window(512, 640, 90); + let mut session = OverlaySession::new(); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(monitor); @@ -17195,9 +17213,9 @@ mod tests { let events = Arc::new([(1, event_at, 150.0, 160.0, -4.0, true, false)]); let base_frame = make_scroll_capture_window(&document, 3, 0, 5); let latest_frame = make_scroll_capture_window(&document, 3, 1, 5); - let mut session = OverlaySession::new(); let scroll_session = ScrollSession::new(base_frame.clone(), 320).unwrap(); let committed_preview = scroll_session.preview_image().clone(); + let mut session = OverlaySession::new(); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(monitor); @@ -17251,10 +17269,12 @@ mod tests { let moved_frame = make_scroll_capture_window(&document, 3, 1, 5); let mut session = OverlaySession::new(); let mut scroll_session = ScrollSession::new(base_frame, 320).unwrap(); + assert!(matches!( scroll_session.observe_downward_sample(moved_frame.clone()).unwrap(), ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 1 } )); + let committed_preview = scroll_session.preview_image().clone(); session.scroll_capture.active = true; @@ -17307,21 +17327,31 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 640); assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 0); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 724); assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); } @@ -17345,20 +17375,26 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(scroll_capture_export_height(&session), 640); session.scroll_capture.last_external_scroll_input_seq = 2; session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; + session.maybe_tick_scroll_capture(); assert!( @@ -17367,6 +17403,7 @@ mod tests { ); drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); assert_eq!(scroll_capture_export_height(&session), 724); } @@ -17396,26 +17433,31 @@ mod tests { session.scroll_capture.session = Some( ScrollSession::new(make_browser_like_worker_capture_window(512, 640, 0), 320).unwrap(), ); + enable_test_worker_scroll_capture_path(&mut session); for expected_top_y in [84_i32, 168, 252] { let mut attempts = 0_u8; + while session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y() < expected_top_y { attempts = attempts.saturating_add(1); + assert!( attempts <= 4, "worker path failed to recover to expected_top_y={expected_top_y}" ); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = session.scroll_capture.last_external_scroll_input_seq.saturating_add(1); session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); session.scroll_capture.last_external_scroll_input_seq = @@ -17423,6 +17465,7 @@ mod tests { session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; + drain_scroll_capture_worker_until_idle(&mut session); } @@ -17451,15 +17494,20 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + session.scroll_capture.last_external_scroll_input_seq = 2; session.scroll_capture.input_direction = Some(ScrollDirection::Down); + drain_scroll_capture_worker_until_idle(&mut session); assert_eq!(scroll_capture_export_height(&session), 820); @@ -17488,18 +17536,22 @@ mod tests { session.scroll_capture.session = Some( ScrollSession::new(make_browser_like_worker_capture_window(512, 640, 0), 320).unwrap(), ); + enable_test_worker_scroll_capture_path(&mut session); for (step, expected_top_y) in [84_i32, 168, 252].into_iter().enumerate() { set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); session.scroll_capture.last_external_scroll_input_seq = (step as u64) + 2; session.scroll_capture.input_direction = Some(ScrollDirection::Down); + drain_scroll_capture_worker_until_idle(&mut session); assert_eq!(session.scroll_capture.inflight_request_id, None); @@ -17531,15 +17583,20 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + session.scroll_capture.last_external_scroll_input_seq = 2; session.scroll_capture.input_direction = Some(ScrollDirection::Up); + drain_scroll_capture_worker_until_idle(&mut session); assert_eq!(scroll_capture_export_height(&session), 640); @@ -17564,14 +17621,19 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.inflight_request_id, None); assert_eq!(scroll_capture_export_height(&session), 640); @@ -17579,6 +17641,7 @@ mod tests { session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; + session.maybe_tick_scroll_capture(); assert!( @@ -17587,6 +17650,7 @@ mod tests { ); drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); assert_eq!(scroll_capture_export_height(&session), 724); } @@ -17613,14 +17677,19 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); - enable_test_worker_scroll_capture_path(&mut session); + enable_test_worker_scroll_capture_path(&mut session); set_scroll_capture_input(&mut session, ScrollDirection::Down); + session.scroll_capture.last_external_scroll_input_seq = 1; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); assert_eq!(scroll_capture_export_height(&session), 724); @@ -17629,14 +17698,19 @@ mod tests { session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.inflight_request_id, None); assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 84); assert_eq!(scroll_capture_export_height(&session), 724); session.maybe_tick_scroll_capture(); + assert!( session.scroll_capture.inflight_request_id.is_none(), "duplicate committed worker frame should back off instead of immediately re-requesting" @@ -17647,9 +17721,13 @@ mod tests { session.scroll_capture.input_direction_at = Some(Instant::now()); session.scroll_capture.input_gesture_active = true; session.scroll_capture.next_sample_at = Some(Instant::now() - Duration::from_millis(1)); + session.maybe_tick_scroll_capture(); + assert!(session.scroll_capture.inflight_request_id.is_some()); + drain_scroll_capture_worker_until_idle(&mut session); + assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 168); assert_eq!(scroll_capture_export_height(&session), 808); } @@ -17711,7 +17789,9 @@ mod tests { session.scroll_capture.monitor = Some(monitor); session.scroll_capture.capture_rect_pixels = Some(rect); session.scroll_capture.live_stream = Some(MacLiveFrameStream::new()); + enable_test_worker_scroll_capture_path(&mut session); + session.set_external_scroll_input_drain_reader(Arc::new({ let events = Arc::clone(&events); @@ -17735,8 +17815,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn force_stream_refresh_stays_disabled_while_downward_gesture_is_still_active() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(now); @@ -17749,8 +17829,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn stale_stream_refresh_stays_disabled_while_gesture_is_still_active() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_gesture_active = true; session.scroll_capture.last_stream_event_at = Some( @@ -17764,8 +17844,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn stale_stream_refresh_reenables_after_gesture_ends() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_gesture_active = false; @@ -17775,8 +17855,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn stale_stream_refresh_reenables_during_gesture_after_stream_goes_dead() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_gesture_active = true; session.scroll_capture.last_stream_event_at = Some( @@ -17791,8 +17871,8 @@ mod tests { #[test] fn post_stall_burst_search_stays_enabled_during_active_gesture_when_downward_backlog_is_fresh() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); session.scroll_capture.input_direction = Some(ScrollDirection::Down); @@ -17806,8 +17886,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn force_stream_refresh_stays_enabled_for_fresh_pending_downward_motion_after_gesture_end() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = @@ -17821,8 +17901,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn force_stream_refresh_stops_after_downward_input_becomes_stale() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = @@ -17836,8 +17916,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn post_stall_burst_search_stays_enabled_while_fresh_downward_backlog_remains() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); session.scroll_capture.input_direction = Some(ScrollDirection::Down); @@ -17855,8 +17935,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn post_stall_burst_search_arms_for_large_capture_time_gap_even_when_frame_seq_is_contiguous() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(now); @@ -17915,12 +17995,12 @@ mod tests { .collect() } })); + session.test_push_scroll_capture_live_frame(ScrollCaptureLiveFrame { frame_seq: 9, captured_at, image: make_scroll_capture_window(&document, 3, 0, 5), }); - session.test_consume_scroll_capture_backlog(1); assert_eq!(session.scroll_capture.input_direction, Some(ScrollDirection::Down)); @@ -17931,8 +18011,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn post_stall_burst_search_does_not_arm_for_small_capture_time_gap() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.input_direction = Some(ScrollDirection::Down); session.scroll_capture.input_direction_at = Some(now); @@ -17949,8 +18029,8 @@ mod tests { #[cfg(target_os = "macos")] #[test] fn post_stall_burst_search_stops_after_downward_backlog_goes_stale() { - let mut session = OverlaySession::new(); let now = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.pending_post_stall_burst_after_seq = Some(80); session.scroll_capture.input_direction = Some(ScrollDirection::Down); @@ -18244,7 +18324,6 @@ mod tests { growth_rows: 1, }) ); - assert_eq!( session.scroll_capture.session.as_ref().unwrap().export_image().height(), height_after_first_append + 1 diff --git a/packages/rsnap-overlay/src/overlay/replay_support.rs b/packages/rsnap-overlay/src/overlay/replay_support.rs index 7785d402..47455aad 100644 --- a/packages/rsnap-overlay/src/overlay/replay_support.rs +++ b/packages/rsnap-overlay/src/overlay/replay_support.rs @@ -1,20 +1,25 @@ +use std::path::PathBuf; use std::{ path::Path, time::{Duration, Instant}, }; -use color_eyre::eyre::{Result, WrapErr, eyre}; -use image::RgbaImage; +use color_eyre::eyre::{self, Result, WrapErr}; +use image::{self, RgbaImage}; use serde::Serialize; -use super::{ +use crate::overlay::trace_recording::ScrollCaptureTraceDirection; +use crate::overlay::trace_recording::ScrollCaptureTraceFrameEntry; +use crate::overlay::trace_recording::ScrollCaptureTraceInputEntry; +use crate::overlay::{ GlobalPoint, MonitorRect, OverlaySession, RectPoints, ScrollCaptureFrameSource, ScrollDirection, ScrollObserveOutcome, ScrollSession, trace_recording::{ - LoadedScrollCaptureLiveTrace, ScrollCaptureLiveTraceEntry, ScrollCaptureTraceFrameSource, + LoadedScrollCaptureLiveTrace, ScrollCaptureLiveTraceEntry, ScrollCaptureTraceRecordedOutcome, }, }; +use crate::scroll_capture::ScrollCommitTelemetry; #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] /// Public replay outcome surface that does not leak private scroll-capture internals. @@ -31,7 +36,6 @@ pub enum ScrollCaptureReplayOutcome { growth_rows: u32, }, } - impl From for ScrollCaptureReplayOutcome { fn from(value: ScrollObserveOutcome) -> Self { match value { @@ -58,23 +62,87 @@ pub enum RecordedScrollCaptureReplayMode { ForceWorkerPairwise, } -fn classify_replayed_outcome( - outcome: ScrollObserveOutcome, - previous_replayed_export_height: Option, - replayed_export_height: u32, - previous_replayed_preview_height: Option, - replayed_preview_height: u32, -) -> ScrollCaptureReplayOutcome { - let replayed_outcome: ScrollCaptureReplayOutcome = outcome.into(); +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Semantic failure classes inferred from recorded frame-to-frame motion. +pub enum RecordedScrollCaptureSemanticIssue { + /// Recorded frames moved downward but the recorded outcome did not convert that into growth. + MissedDownwardMotion, + /// Recorded frames moved downward significantly more than the recorded committed growth. + UnderconsumedDownwardMotion, + /// Recorded committed growth exceeded the visible recorded frame-to-frame shift by a large margin. + GrowthExceedsRecordedShift, +} - if replayed_outcome == ScrollCaptureReplayOutcome::NoChange - && previous_replayed_export_height == Some(replayed_export_height) - && previous_replayed_preview_height - .is_some_and(|previous| replayed_preview_height > previous) - { - ScrollCaptureReplayOutcome::PreviewUpdated - } else { - replayed_outcome +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Public frame-source surface for one replayed live-trace step. +pub enum RecordedScrollCaptureReplayFrameSource { + Worker { request_id: u64 }, + LiveStream { frame_seq: u64 }, +} +impl From + for RecordedScrollCaptureReplayFrameSource +{ + fn from(value: super::trace_recording::ScrollCaptureTraceFrameSource) -> Self { + match value { + super::trace_recording::ScrollCaptureTraceFrameSource::Worker { request_id } => { + Self::Worker { request_id } + }, + super::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { + Self::LiveStream { frame_seq } + }, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +/// Public recorded-outcome surface for one frame in a replayed live trace. +pub enum RecordedScrollCaptureReplayRecordedOutcome { + /// The live frame did not change preview or export state. + NoChange, + /// The live frame updated preview state without committing growth. + PreviewUpdated, + /// The live frame detected unsupported motion. + Unsupported { + /// Direction recorded during the live session. + direction: &'static str, + }, + /// The live frame committed stitched growth. + Committed { + /// Direction recorded during the live session. + direction: &'static str, + /// Newly appended rows recorded during the live session. + growth_rows: u32, + }, + /// The live frame recorded an observation error. + Error { + /// Error string captured during the live session. + message: String, + }, +} +impl From for RecordedScrollCaptureReplayRecordedOutcome { + fn from(value: ScrollCaptureTraceRecordedOutcome) -> Self { + match value { + ScrollCaptureTraceRecordedOutcome::NoChange => Self::NoChange, + ScrollCaptureTraceRecordedOutcome::PreviewUpdated => Self::PreviewUpdated, + ScrollCaptureTraceRecordedOutcome::UnsupportedDirection { direction } => { + Self::Unsupported { + direction: match direction { + ScrollCaptureTraceDirection::Up => "up", + ScrollCaptureTraceDirection::Down => "down", + }, + } + }, + ScrollCaptureTraceRecordedOutcome::Committed { direction, growth_rows } => { + Self::Committed { + direction: match direction { + ScrollCaptureTraceDirection::Up => "up", + ScrollCaptureTraceDirection::Down => "down", + }, + growth_rows, + } + }, + ScrollCaptureTraceRecordedOutcome::Error { message } => Self::Error { message }, + } } } @@ -86,7 +154,7 @@ pub struct RecordedScrollCaptureReplaySummary { /// Stable trace id from the manifest. pub trace_id: String, /// Manifest path used to load the trace. - pub manifest_path: std::path::PathBuf, + pub manifest_path: PathBuf, /// Final stitched export height after replaying the recorded trace. pub final_export_height: u32, /// Final preview height after replaying the recorded trace. @@ -98,9 +166,9 @@ pub struct RecordedScrollCaptureReplaySummary { /// Final preview height recorded during the live session, when present. pub recorded_final_preview_height: Option, /// Final recorded live-trace preview artifact, when present. - pub final_preview_path: Option, + pub final_preview_path: Option, /// Final recorded live-trace export artifact, when present. - pub final_export_path: Option, + pub final_export_path: Option, /// First frame where recorded and replayed outcomes diverged. pub first_outcome_divergence_frame: Option, /// First frame where replayed export height drifted from the recorded live trace. @@ -252,90 +320,21 @@ pub struct RecordedScrollCaptureReplayStepResult { pub semantic_issue: Option, } -#[derive(Clone, Debug, Eq, PartialEq, Serialize)] -/// Semantic failure classes inferred from recorded frame-to-frame motion. -pub enum RecordedScrollCaptureSemanticIssue { - /// Recorded frames moved downward but the recorded outcome did not convert that into growth. - MissedDownwardMotion, - /// Recorded frames moved downward significantly more than the recorded committed growth. - UnderconsumedDownwardMotion, - /// Recorded committed growth exceeded the visible recorded frame-to-frame shift by a large margin. - GrowthExceedsRecordedShift, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize)] -/// Public frame-source surface for one replayed live-trace step. -pub enum RecordedScrollCaptureReplayFrameSource { - Worker { request_id: u64 }, - LiveStream { frame_seq: u64 }, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize)] -/// Public recorded-outcome surface for one frame in a replayed live trace. -pub enum RecordedScrollCaptureReplayRecordedOutcome { - /// The live frame did not change preview or export state. - NoChange, - /// The live frame updated preview state without committing growth. - PreviewUpdated, - /// The live frame detected unsupported motion. - Unsupported { - /// Direction recorded during the live session. - direction: &'static str, - }, - /// The live frame committed stitched growth. - Committed { - /// Direction recorded during the live session. - direction: &'static str, - /// Newly appended rows recorded during the live session. - growth_rows: u32, - }, - /// The live frame recorded an observation error. - Error { - /// Error string captured during the live session. - message: String, - }, -} - -impl From for RecordedScrollCaptureReplayRecordedOutcome { - fn from(value: ScrollCaptureTraceRecordedOutcome) -> Self { - match value { - ScrollCaptureTraceRecordedOutcome::NoChange => Self::NoChange, - ScrollCaptureTraceRecordedOutcome::PreviewUpdated => Self::PreviewUpdated, - ScrollCaptureTraceRecordedOutcome::UnsupportedDirection { direction } => { - Self::Unsupported { - direction: match direction { - super::trace_recording::ScrollCaptureTraceDirection::Up => "up", - super::trace_recording::ScrollCaptureTraceDirection::Down => "down", - }, - } - }, - ScrollCaptureTraceRecordedOutcome::Committed { direction, growth_rows } => { - Self::Committed { - direction: match direction { - super::trace_recording::ScrollCaptureTraceDirection::Up => "up", - super::trace_recording::ScrollCaptureTraceDirection::Down => "down", - }, - growth_rows, - } - }, - ScrollCaptureTraceRecordedOutcome::Error { message } => Self::Error { message }, - } - } -} - -impl From - for RecordedScrollCaptureReplayFrameSource -{ - fn from(value: super::trace_recording::ScrollCaptureTraceFrameSource) -> Self { - match value { - super::trace_recording::ScrollCaptureTraceFrameSource::Worker { request_id } => { - Self::Worker { request_id } - }, - super::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { - Self::LiveStream { frame_seq } - }, - } - } +#[derive(Default)] +struct ReplayStats { + step_results: Vec, + previous_recorded_export_height: Option, + previous_recorded_preview_height: Option, + previous_replayed_export_height: Option, + previous_replayed_preview_height: Option, + previous_live_frame_seq: Option, + previous_recorded_frame: Option, + max_recorded_export_jump: u32, + max_recorded_preview_jump: u32, + max_replayed_export_jump: u32, + max_replayed_preview_jump: u32, + max_recorded_committed_growth_rows: u32, + max_replayed_committed_growth_rows: u32, } /// Replays one recorded live trace through shipping overlay logic. @@ -360,36 +359,42 @@ pub fn replay_recorded_scroll_capture_trace_with_mode( finalize_replay_summary(trace, &session, replay_stats, replay_mode) } -#[derive(Default)] -struct ReplayStats { - step_results: Vec, - previous_recorded_export_height: Option, - previous_recorded_preview_height: Option, +fn classify_replayed_outcome( + outcome: ScrollObserveOutcome, previous_replayed_export_height: Option, + replayed_export_height: u32, previous_replayed_preview_height: Option, - previous_live_frame_seq: Option, - previous_recorded_frame: Option, - max_recorded_export_jump: u32, - max_recorded_preview_jump: u32, - max_replayed_export_jump: u32, - max_replayed_preview_jump: u32, - max_recorded_committed_growth_rows: u32, - max_replayed_committed_growth_rows: u32, + replayed_preview_height: u32, +) -> ScrollCaptureReplayOutcome { + let replayed_outcome: ScrollCaptureReplayOutcome = outcome.into(); + + if replayed_outcome == ScrollCaptureReplayOutcome::NoChange + && previous_replayed_export_height == Some(replayed_export_height) + && previous_replayed_preview_height + .is_some_and(|previous| replayed_preview_height > previous) + { + ScrollCaptureReplayOutcome::PreviewUpdated + } else { + replayed_outcome + } } fn initialize_replay_session( trace: &LoadedScrollCaptureLiveTrace, ) -> Result<(OverlaySession, Instant)> { - let mut session = OverlaySession::new(); let started_at = Instant::now(); + let mut session = OverlaySession::new(); session.scroll_capture.active = true; session.scroll_capture.monitor = Some(replay_monitor_from_trace(trace)); session.scroll_capture.capture_rect_pixels = Some(replay_capture_rect_from_trace(trace)); session.scroll_capture.session = Some(ScrollSession::new(trace.base_frame.clone(), trace.manifest.preview_width_px)?); + session.refresh_scroll_preview_committed_image(); + session.scroll_capture.preview_latest_frame = Some(trace.base_frame.clone()); + session.refresh_scroll_preview_display_image(); Ok((session, started_at)) @@ -426,7 +431,7 @@ fn replay_trace_entries( fn apply_replayed_input( session: &mut OverlaySession, - input: &super::trace_recording::ScrollCaptureTraceInputEntry, + input: &ScrollCaptureTraceInputEntry, started_at: Instant, ) { session.apply_external_scroll_input_delta_y( @@ -440,10 +445,11 @@ fn apply_replayed_input( session.refresh_scroll_preview_display_image(); } +#[allow(clippy::too_many_lines)] fn replay_frame_entry( trace: &LoadedScrollCaptureLiveTrace, session: &mut OverlaySession, - frame: &super::trace_recording::ScrollCaptureTraceFrameEntry, + frame: &ScrollCaptureTraceFrameEntry, started_at: Instant, replay_mode: RecordedScrollCaptureReplayMode, replay_stats: &mut ReplayStats, @@ -452,6 +458,7 @@ fn replay_frame_entry( frame.snapshot_after.export_dimensions.map(|dimensions| dimensions[1]); let recorded_preview_height = frame.snapshot_after.preview_dimensions.map(|dimensions| dimensions[1]); + update_recorded_height_jumps(replay_stats, recorded_export_height, recorded_preview_height); let image = image::open(trace.resolve_frame_path(&frame.frame_path)) @@ -464,19 +471,22 @@ fn replay_frame_entry( let observed_at = started_at + Duration::from_millis(frame.observed_at_ms); let outcome = match replay_mode { RecordedScrollCaptureReplayMode::RecordedSource => match frame.frame_source { - ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => session - .replay_recorded_live_stream_frame( - image.clone(), - frame_seq, - observed_at, - frame.allow_stale_input, - ), - ScrollCaptureTraceFrameSource::Worker { .. } => session.handle_scroll_capture_frame( + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { + frame_seq, + } => session.replay_recorded_live_stream_frame( image.clone(), - replay_frame_source(frame.frame_source), - frame.allow_stale_input, + frame_seq, observed_at, + frame.allow_stale_input, ), + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::Worker { .. } => { + session.handle_scroll_capture_frame( + image.clone(), + replay_frame_source(frame.frame_source), + frame.allow_stale_input, + observed_at, + ) + }, }, RecordedScrollCaptureReplayMode::ForceWorkerPairwise => session .handle_scroll_capture_frame( @@ -490,13 +500,16 @@ fn replay_frame_entry( } .transpose()? .ok_or_else(|| { - eyre!( + eyre::eyre!( "recorded trace frame {} did not observe because the session vanished", frame.frame_path ) })?; let active_session = session.scroll_capture.session.as_ref().ok_or_else(|| { - eyre!("scroll-capture session missing after replaying recorded frame {}", frame.frame_path) + eyre::eyre!( + "scroll-capture session missing after replaying recorded frame {}", + frame.frame_path + ) })?; let telemetry = active_session.commit_telemetry(); let frame_source: RecordedScrollCaptureReplayFrameSource = frame.frame_source.into(); @@ -527,8 +540,51 @@ fn replay_frame_entry( replay_stats.max_replayed_committed_growth_rows = replay_stats.max_replayed_committed_growth_rows.max(growth_rows); } + update_replayed_height_jumps(replay_stats, replayed_export_height, replayed_preview_height); + push_replay_step_result( + replay_stats, + session, + active_session, + &telemetry, + frame, + frame_source, + live_frame_gap, + recorded_outcome, + replayed_outcome, + recorded_export_height, + recorded_preview_height, + replayed_export_height, + replayed_preview_height, + replayed_session_preview_height, + recorded_estimated_downward_shift_rows, + semantic_issue, + ); + + replay_stats.previous_recorded_frame = Some(image); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn push_replay_step_result( + replay_stats: &mut ReplayStats, + session: &OverlaySession, + active_session: &ScrollSession, + telemetry: &ScrollCommitTelemetry, + frame: &ScrollCaptureTraceFrameEntry, + frame_source: RecordedScrollCaptureReplayFrameSource, + live_frame_gap: Option, + recorded_outcome: RecordedScrollCaptureReplayRecordedOutcome, + replayed_outcome: ScrollCaptureReplayOutcome, + recorded_export_height: Option, + recorded_preview_height: Option, + replayed_export_height: u32, + replayed_preview_height: u32, + replayed_session_preview_height: u32, + recorded_estimated_downward_shift_rows: Option, + semantic_issue: Option, +) { replay_stats.step_results.push(RecordedScrollCaptureReplayStepResult { frame_index: replay_stats.step_results.len(), frame_path: frame.frame_path.clone(), @@ -571,9 +627,11 @@ fn replay_frame_entry( replayed_downward_viewport_candidate_count: telemetry .last_downward_viewport_candidate_count, replayed_downward_viewport_candidates_before_prune: telemetry - .last_downward_viewport_candidates_before_prune, + .last_downward_viewport_candidates_before_prune + .clone(), replayed_downward_viewport_candidates_after_prune: telemetry - .last_downward_viewport_candidates_after_prune, + .last_downward_viewport_candidates_after_prune + .clone(), replayed_sample_eval_last_motion_rows_hint: telemetry.sample_eval_last_motion_rows_hint, replayed_sample_eval_transient_motion_rows_hint: telemetry .sample_eval_transient_motion_rows_hint, @@ -621,7 +679,7 @@ fn replay_frame_entry( .scroll_capture .retained_overlay_preview_image .as_ref() - .map(image::RgbaImage::height), + .map(RgbaImage::height), replayed_retained_overlay_preview_motion_rows_hint: session .scroll_capture .retained_overlay_preview_motion_rows_hint, @@ -637,9 +695,6 @@ fn replay_frame_entry( recorded_estimated_downward_shift_rows, semantic_issue, }); - replay_stats.previous_recorded_frame = Some(image); - - Ok(()) } fn update_recorded_height_jumps( @@ -653,15 +708,16 @@ fn update_recorded_height_jumps( .max_recorded_export_jump .max(recorded_export_height.saturating_sub(previous)); } + replay_stats.previous_recorded_export_height = Some(recorded_export_height); } - if let Some(recorded_preview_height) = recorded_preview_height { if let Some(previous) = replay_stats.previous_recorded_preview_height { replay_stats.max_recorded_preview_jump = replay_stats .max_recorded_preview_jump .max(recorded_preview_height.saturating_sub(previous)); } + replay_stats.previous_recorded_preview_height = Some(recorded_preview_height); } } @@ -676,6 +732,7 @@ fn update_replayed_height_jumps( .max_replayed_export_jump .max(replayed_export_height.saturating_sub(previous)); } + replay_stats.previous_replayed_export_height = Some(replayed_export_height); if let Some(previous) = replay_stats.previous_replayed_preview_height { @@ -683,6 +740,7 @@ fn update_replayed_height_jumps( .max_replayed_preview_jump .max(replayed_preview_height.saturating_sub(previous)); } + replay_stats.previous_replayed_preview_height = Some(replayed_preview_height); } @@ -696,7 +754,9 @@ fn update_live_frame_gap( .previous_live_frame_seq .map(|previous| frame_seq.saturating_sub(previous)) .unwrap_or(1); + replay_stats.previous_live_frame_seq = Some(frame_seq); + Some(gap) }, RecordedScrollCaptureReplayFrameSource::Worker { .. } => None, @@ -710,7 +770,7 @@ fn finalize_replay_summary( replay_mode: RecordedScrollCaptureReplayMode, ) -> Result { let final_session = session.scroll_capture.session.as_ref().ok_or_else(|| { - eyre!("scroll-capture session missing after replaying recorded live trace") + eyre::eyre!("scroll-capture session missing after replaying recorded live trace") })?; let first_outcome_divergence_frame = replay_stats .step_results @@ -819,7 +879,9 @@ fn estimate_recorded_downward_shift_rows(previous: &RgbaImage, current: &RgbaIma if previous.dimensions() != current.dimensions() { return None; } + let (width, height) = previous.dimensions(); + if width < 2 || height < 3 { return None; } @@ -830,7 +892,6 @@ fn estimate_recorded_downward_shift_rows(previous: &RgbaImage, current: &RgbaIma let x_step = ((end_x.saturating_sub(start_x)) / 48).max(1); let y_step = 2_u32; let max_shift = height.saturating_sub(1).min(96); - let mut best_shift = 0_u32; let mut best_score = overlap_abs_diff(previous, current, 0, start_x, end_x, x_step, y_step)?; @@ -840,6 +901,7 @@ fn estimate_recorded_downward_shift_rows(previous: &RgbaImage, current: &RgbaIma else { continue; }; + if score < best_score { best_score = score; best_shift = shift; @@ -859,10 +921,13 @@ fn overlap_abs_diff( y_step: u32, ) -> Option { let height = previous.height(); + if shift >= height { return None; } + let overlap_height = height - shift; + if overlap_height < 2 { return None; } @@ -870,17 +935,21 @@ fn overlap_abs_diff( let mut sum = 0_u64; let mut samples = 0_u64; let mut y = 0_u32; + while y < overlap_height { let mut x = start_x; + while x < end_x { let prev = previous.get_pixel(x, y + shift); let curr = current.get_pixel(x, y); let prev_luma = u16::from(prev[0]) + u16::from(prev[1]) + u16::from(prev[2]); let curr_luma = u16::from(curr[0]) + u16::from(curr[1]) + u16::from(curr[2]); + sum += u64::from(prev_luma.abs_diff(curr_luma)); samples += 1; x = x.saturating_add(x_step); } + y = y.saturating_add(y_step); } @@ -896,6 +965,7 @@ fn classify_recorded_semantic_issue( recorded_estimated_downward_shift_rows: Option, ) -> Option { let shift = recorded_estimated_downward_shift_rows?; + if shift < 4 { return None; } @@ -922,25 +992,32 @@ fn classify_recorded_semantic_issue( } #[cfg(target_os = "macos")] -fn replay_frame_source(frame_source: ScrollCaptureTraceFrameSource) -> ScrollCaptureFrameSource { +fn replay_frame_source( + frame_source: crate::overlay::trace_recording::ScrollCaptureTraceFrameSource, +) -> ScrollCaptureFrameSource { match frame_source { - ScrollCaptureTraceFrameSource::Worker { .. } => { + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::Worker { .. } => { unreachable!("macOS live traces should not contain worker-backed scroll frames") }, - ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { - ScrollCaptureFrameSource::LiveStream { frame_seq } - }, + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { + frame_seq, + } => ScrollCaptureFrameSource::LiveStream { frame_seq }, } } #[cfg(not(target_os = "macos"))] -fn replay_frame_source(frame_source: ScrollCaptureTraceFrameSource) -> ScrollCaptureFrameSource { +fn replay_frame_source( + frame_source: crate::overlay::trace_recording::ScrollCaptureTraceFrameSource, +) -> ScrollCaptureFrameSource { match frame_source { - ScrollCaptureTraceFrameSource::Worker { request_id } => { + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::Worker { request_id } => { ScrollCaptureFrameSource::Worker { request_id } }, - ScrollCaptureTraceFrameSource::LiveStream { frame_seq } => { + crate::overlay::trace_recording::ScrollCaptureTraceFrameSource::LiveStream { + frame_seq, + } => { let _ = frame_seq; + unreachable!("non-macOS replay should not receive live-stream scroll frames") }, } @@ -1011,6 +1088,7 @@ fn replay_capture_rect_from_trace(trace: &LoadedScrollCaptureLiveTrace) -> RectP #[cfg(test)] mod tests { + use std::env; use std::{ fs, path::PathBuf, @@ -1020,10 +1098,7 @@ mod tests { use image::{Rgba, RgbaImage}; - use super::{ - RecordedScrollCaptureReplayMode, replay_recorded_scroll_capture_trace, - replay_recorded_scroll_capture_trace_with_mode, - }; + use crate::overlay::replay_support::{self, RecordedScrollCaptureReplayMode}; use crate::overlay::{ GlobalPoint, MonitorRect, OverlaySession, RectPoints, ScrollCaptureFrameSource, trace_recording::{ @@ -1034,7 +1109,7 @@ mod tests { use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; fn temp_trace_root() -> PathBuf { - let root = std::env::temp_dir().join(format!( + let root = env::temp_dir().join(format!( "rsnap-recorded-trace-replay-test-{}-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis(), process::id() @@ -1155,6 +1230,7 @@ mod tests { Some(2), ), }); + let outcome = session .observe_scroll_capture_frame_at( next_frame.clone(), @@ -1163,6 +1239,7 @@ mod tests { .transpose() .unwrap() .unwrap(); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { frame: &next_frame, source: ScrollCaptureFrameSource::LiveStream { frame_seq: 9 }, @@ -1187,7 +1264,7 @@ mod tests { drop(recorder); - let summary = replay_recorded_scroll_capture_trace(&manifest_path).unwrap(); + let summary = replay_support::replay_recorded_scroll_capture_trace(&manifest_path).unwrap(); assert_eq!(summary.step_results.len(), 1); assert_eq!( @@ -1258,6 +1335,7 @@ mod tests { Some(2), ), }); + let outcome = session .observe_scroll_capture_frame_at( next_frame.clone(), @@ -1266,6 +1344,7 @@ mod tests { .transpose() .unwrap() .unwrap(); + recorder.record_frame_observation(ScrollCaptureTraceFrameRecord { frame: &next_frame, source: ScrollCaptureFrameSource::LiveStream { frame_seq: 9 }, @@ -1295,8 +1374,7 @@ mod tests { else { panic!("expected recorded-source setup to commit one downward growth step"); }; - - let summary = replay_recorded_scroll_capture_trace_with_mode( + let summary = replay_support::replay_recorded_scroll_capture_trace_with_mode( &manifest_path, RecordedScrollCaptureReplayMode::ForceWorkerPairwise, ) diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index 8d3db240..a9c2f86f 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -3,8 +3,10 @@ use std::time::{Duration, Instant}; use color_eyre::Result; use image::RgbaImage; -use crate::overlay::SCROLL_CAPTURE_SAMPLE_INTERVAL; +#[cfg(target_os = "macos")] +use crate::live_frame_stream_macos::MacLiveFrameStream; use crate::overlay::SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL; +use crate::overlay::SCROLL_CAPTURE_SAMPLE_INTERVAL; #[cfg(target_os = "macos")] use crate::overlay::session_state::ScrollCaptureLiveFrame; #[cfg(target_os = "macos")] @@ -19,10 +21,9 @@ use crate::overlay::{ OverlayControl, OverlaySession, ScrollCaptureFrameSource, ScrollCaptureTraceFrameRecord, ScrollCaptureTraceInputRecord, ScrollObserveOutcome, ScrollSession, }; -#[cfg(target_os = "macos")] use crate::scroll_capture::ScrollDirection; #[cfg(target_os = "macos")] -use crate::scroll_capture::{scroll_capture_fingerprint, scroll_capture_fingerprint_delta}; +use crate::scroll_capture::{self}; use crate::worker::WorkerRequestSendError; impl OverlaySession { @@ -52,20 +53,25 @@ impl OverlaySession { self.sync_scroll_overlay_mouse_passthrough_window(now); self.drain_external_scroll_input_events_through(now); + if self.should_use_scroll_capture_worker_sampling() { self.request_scroll_capture_worker_sample_at(now); return; } + self.poll_scroll_stream_fallback_if_due(now); + if self.scroll_capture.live_stream.is_some() - && self.scroll_capture.last_stream_poll_at.map_or(true, |last| { + && self.scroll_capture.last_stream_poll_at.is_none_or(|last| { now.saturating_duration_since(last) >= SCROLL_CAPTURE_STREAM_POLL_INTERVAL }) { self.scroll_capture.last_stream_poll_at = Some(now); + let _ = self.try_consume_scroll_stream_frame(); } } + #[cfg(not(target_os = "macos"))] { self.request_scroll_capture_worker_sample_at(Instant::now()); @@ -142,11 +148,13 @@ impl OverlaySession { && self.scroll_capture.input_direction_at.is_some_and(|input_direction_at| { now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS }); + if !fresh_downward_input { return; } self.scroll_capture.next_sample_at = Some(now); + tracing::info!( op = "scroll_capture.worker_retry_scheduled_immediately", reason = why, @@ -168,11 +176,13 @@ impl OverlaySession { && self.scroll_capture.input_direction_at.is_some_and(|input_direction_at| { now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS }); + if !fresh_downward_input { return; } self.scroll_capture.next_sample_at = Some(now + delay); + tracing::info!( op = "scroll_capture.worker_retry_scheduled_with_backoff", reason = why, @@ -205,70 +215,57 @@ impl OverlaySession { return false; }; let last_frame_seq = self.scroll_capture.last_stream_frame_seq; - let log_stream_frame_empty = |query_ms: u64, refresh_scheduled: bool| { - if query_ms >= 16 { - tracing::warn!( - op = "scroll_capture.stream_frame_query_slow", - last_frame_seq, - query_ms, - refresh_scheduled, - stale_refresh_suppressed = !allow_stale_refresh, - force_refresh, - result = "empty", - "Slow nonblocking live-stream query delayed scroll-capture observation." - ); - } - tracing::info!( - op = "scroll_capture.stream_frame_empty", - last_frame_seq, - query_ms, - refresh_scheduled, - stale_refresh_suppressed = !allow_stale_refresh, - force_refresh, - "Did not receive a newer live-stream frame for scroll-capture observation." - ); - }; let Some(frames) = live_stream.ordered_rgba_regions_after_seq_nonblocking( monitor, capture_rect, last_frame_seq, ) else { - let query_ms = - u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); - let refresh_scheduled = if allow_stale_refresh { - live_stream.refresh_monitor_nonblocking_if_stale( - monitor, - last_frame_seq, - force_refresh, - ) - } else { - false - }; + let (query_ms, refresh_scheduled) = Self::query_empty_scroll_stream_result( + live_stream, + monitor, + last_frame_seq, + query_started_at, + allow_stale_refresh, + force_refresh, + ); + let _ = live_stream; + if refresh_scheduled && fresh_downward_backlog { self.scroll_capture.pending_post_stall_burst_after_seq = Some(last_frame_seq); } - log_stream_frame_empty(query_ms, refresh_scheduled); + self.log_empty_scroll_stream_query( + last_frame_seq, + query_ms, + refresh_scheduled, + allow_stale_refresh, + force_refresh, + ); return false; }; let Some(newest_frame_seq) = frames.last().map(|frame| frame.frame_seq) else { - let query_ms = - u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); - let refresh_scheduled = if allow_stale_refresh { - live_stream.refresh_monitor_nonblocking_if_stale( - monitor, - last_frame_seq, - force_refresh, - ) - } else { - false - }; + let (query_ms, refresh_scheduled) = Self::query_empty_scroll_stream_result( + live_stream, + monitor, + last_frame_seq, + query_started_at, + allow_stale_refresh, + force_refresh, + ); + let _ = live_stream; + if refresh_scheduled && fresh_downward_backlog { self.scroll_capture.pending_post_stall_burst_after_seq = Some(last_frame_seq); } - log_stream_frame_empty(query_ms, refresh_scheduled); + self.log_empty_scroll_stream_query( + last_frame_seq, + query_ms, + refresh_scheduled, + allow_stale_refresh, + force_refresh, + ); return false; }; @@ -309,18 +306,72 @@ impl OverlaySession { true } + #[cfg(target_os = "macos")] + #[allow(clippy::too_many_arguments)] + fn query_empty_scroll_stream_result( + live_stream: &mut MacLiveFrameStream, + monitor: MonitorRect, + last_frame_seq: u64, + query_started_at: Instant, + allow_stale_refresh: bool, + force_refresh: bool, + ) -> (u64, bool) { + let query_ms = u64::try_from(query_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let refresh_scheduled = if allow_stale_refresh { + live_stream.refresh_monitor_nonblocking_if_stale(monitor, last_frame_seq, force_refresh) + } else { + false + }; + + (query_ms, refresh_scheduled) + } + + #[cfg(target_os = "macos")] + fn log_empty_scroll_stream_query( + &self, + last_frame_seq: u64, + query_ms: u64, + refresh_scheduled: bool, + allow_stale_refresh: bool, + force_refresh: bool, + ) { + if query_ms >= 16 { + tracing::warn!( + op = "scroll_capture.stream_frame_query_slow", + last_frame_seq, + query_ms, + refresh_scheduled, + stale_refresh_suppressed = !allow_stale_refresh, + force_refresh, + result = "empty", + "Slow nonblocking live-stream query delayed scroll-capture observation." + ); + } + + tracing::info!( + op = "scroll_capture.stream_frame_empty", + last_frame_seq, + query_ms, + refresh_scheduled, + stale_refresh_suppressed = !allow_stale_refresh, + force_refresh, + "Did not receive a newer live-stream frame for scroll-capture observation." + ); + } + #[cfg(target_os = "macos")] /// Consumes any queued macOS live-stream frames for scroll capture. pub fn handle_scroll_stream_frame_ready(&mut self) -> OverlayControl { if !cfg!(test) { return OverlayControl::Continue; } - if self.scroll_capture.active && !self.scroll_capture.paused { if self.should_use_scroll_capture_worker_sampling() { return OverlayControl::Continue; } + let _ = self.try_consume_scroll_stream_frame(); + self.consume_scroll_capture_backlog(usize::MAX); } @@ -335,11 +386,13 @@ impl OverlaySession { self.sync_scroll_overlay_mouse_passthrough_window(now); self.drain_external_scroll_input_events_through(now); + if self.should_use_scroll_capture_worker_sampling() { self.request_scroll_capture_worker_sample_at(now); return OverlayControl::Continue; } + self.poll_scroll_stream_fallback_if_due(now); self.consume_scroll_capture_backlog(usize::MAX); } @@ -397,7 +450,9 @@ impl OverlaySession { gesture_ended, through, ); + let snapshot_after = self.scroll_capture_trace_snapshot_at(through); + if let Some(trace_recorder) = self.scroll_capture.trace_recorder.as_mut() { trace_recorder.record_replayed_input(ScrollCaptureTraceInputRecord { seq, @@ -410,6 +465,7 @@ impl OverlaySession { snapshot_after, }); } + if !self.should_use_scroll_capture_worker_sampling() { self.refresh_live_stream_stale_grace_for_external_input(seq); } @@ -440,13 +496,14 @@ impl OverlaySession { &mut self, frame: &ScrollCaptureLiveFrame, ) -> bool { - let fingerprint = scroll_capture_fingerprint(&frame.image); + let fingerprint = scroll_capture::scroll_capture_fingerprint(&frame.image); let is_distinct = - self.scroll_capture.last_stream_frame_fingerprint.as_ref().map_or(true, |previous| { - scroll_capture_fingerprint_delta(previous, &fingerprint) > 0 + self.scroll_capture.last_stream_frame_fingerprint.as_ref().is_none_or(|previous| { + scroll_capture::scroll_capture_fingerprint_delta(previous, &fingerprint) > 0 }); self.scroll_capture.last_stream_frame_fingerprint = Some(fingerprint); + if is_distinct { self.scroll_capture.last_stream_event_at = Some(frame.captured_at); self.scroll_capture.consecutive_identical_stream_frames = 0; @@ -486,7 +543,6 @@ impl OverlaySession { let Some(live_stream) = self.scroll_capture.live_stream.as_ref() else { return; }; - let refresh_scheduled = live_stream.refresh_monitor_nonblocking_if_stale(monitor, frame_seq, true); @@ -496,6 +552,7 @@ impl OverlaySession { self.scroll_capture.last_duplicate_stream_refresh_at = Some(observation_at); self.scroll_capture.pending_post_stall_burst_after_seq = Some(frame_seq); + tracing::info!( op = "scroll_capture.duplicate_frame_refresh_scheduled", frame_seq, @@ -508,9 +565,11 @@ impl OverlaySession { #[cfg(target_os = "macos")] fn push_scroll_capture_live_frame(&mut self, frame: ScrollCaptureLiveFrame) { let backlog = &mut self.scroll_capture.live_stream_backlog; + if backlog.len() >= super::SCROLL_CAPTURE_STREAM_BACKLOG_MAX_FRAMES { backlog.pop_front(); } + backlog.push_back(frame); } @@ -522,16 +581,21 @@ impl OverlaySession { #[cfg(target_os = "macos")] fn consume_scroll_capture_backlog(&mut self, max_frames: usize) { let mut consumed = 0; + while consumed < max_frames { let Some(frame) = self.scroll_capture.live_stream_backlog.pop_front() else { break; }; + self.drain_external_scroll_input_events_through(frame.captured_at); + let arm_time_gap_burst = self.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(frame.captured_at); + if arm_time_gap_burst { self.scroll_capture.pending_post_stall_burst_after_seq = Some(frame.frame_seq.saturating_sub(1)); + tracing::info!( op = "scroll_capture.post_stall_burst_search_armed_for_time_gap", frame_seq = frame.frame_seq, @@ -541,7 +605,9 @@ impl OverlaySession { "Armed a burst registration window because the next live-stream frame arrived after a large capture-time gap while fresh downward backlog remained." ); } + let _is_distinct = self.note_scroll_capture_live_stream_frame_activity(&frame); + self.scroll_capture.last_stream_frame_seq = frame.frame_seq; let _ = self.handle_scroll_capture_frame( @@ -550,7 +616,9 @@ impl OverlaySession { false, frame.captured_at, ); + self.scroll_capture.last_consumed_stream_frame_captured_at = Some(frame.captured_at); + self.maybe_schedule_duplicate_stream_refresh(frame.frame_seq, frame.captured_at); consumed += 1; @@ -578,14 +646,18 @@ impl OverlaySession { let frame_for_activity = ScrollCaptureLiveFrame { frame_seq, captured_at: observed_at, image: frame.clone() }; let _ = self.note_scroll_capture_live_stream_frame_activity(&frame_for_activity); + self.scroll_capture.last_stream_frame_seq = frame_seq; + let outcome = self.handle_scroll_capture_frame( frame, ScrollCaptureFrameSource::LiveStream { frame_seq }, allow_stale_input, observed_at, ); + self.scroll_capture.last_consumed_stream_frame_captured_at = Some(observed_at); + self.maybe_schedule_duplicate_stream_refresh(frame_seq, observed_at); outcome @@ -686,11 +758,14 @@ impl OverlaySession { last_external_scroll_input_seq = self.scroll_capture.last_external_scroll_input_seq, "Dropped worker-fed scroll-capture frame because newer external input superseded the request context." ); + self.clear_scroll_capture_inflight_request(); + return; } self.clear_scroll_capture_inflight_request(); + let _ = self.handle_scroll_capture_frame( image, ScrollCaptureFrameSource::Worker { request_id }, @@ -781,10 +856,9 @@ impl OverlaySession { let prior_block_reason = self.scroll_capture_observation_block_reason_at(observation_at); #[cfg(target_os = "macos")] let allow_stale_input = allow_stale_input - || (!allow_stale_input - && prior_block_reason == Some("stale_input") + || prior_block_reason == Some("stale_input") && matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) - && self.consume_live_stream_stale_grace_if_current()); + && self.consume_live_stream_stale_grace_if_current(); #[cfg(not(target_os = "macos"))] let allow_stale_input = allow_stale_input; @@ -840,10 +914,9 @@ impl OverlaySession { allow_post_stall_burst_search, )? }; - if worker_pairwise_path { - if let Ok(outcome) = &outcome { - self.consume_scroll_capture_downward_motion_rows_for_outcome(outcome); - } + + if worker_pairwise_path && let Ok(outcome) = &outcome { + self.consume_scroll_capture_downward_motion_rows_for_outcome(outcome); } if matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) && !allow_post_stall_burst_search @@ -852,10 +925,12 @@ impl OverlaySession { } self.scroll_capture.preview_latest_frame = Some(preview_frame); + self.refresh_scroll_preview_display_image(); self.sync_scroll_preview_segments(); self.request_redraw_scroll_preview_window(); self.handle_scroll_capture_frame_outcome(&outcome, source, frame_px); + let snapshot_after = self.scroll_capture_trace_snapshot_at(observation_at); if let (Some(trace_recorder), Some(trace_frame)) = @@ -875,6 +950,7 @@ impl OverlaySession { Some(outcome) } + #[allow(clippy::too_many_lines)] fn handle_scroll_capture_frame_outcome( &mut self, outcome: &Result, @@ -883,72 +959,10 @@ impl OverlaySession { ) { match outcome { Ok(ScrollObserveOutcome::NoChange) => { - let last_block_reason = - self.scroll_capture.session.as_ref().and_then(ScrollSession::last_block_reason); - tracing::info!( - op = "scroll_capture.frame_observed", - frame_source = source.as_str(), - worker_request_id = ?source.worker_request_id(), - outcome = "no_change", - frame_px = ?frame_px, - input_direction = ?self.scroll_capture.input_direction, - input_gesture_active = self.scroll_capture.input_gesture_active, - last_block_reason = ?last_block_reason, - export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), - "Scroll-capture observed a frame but kept session state unchanged." - ); - if let Some(request_id) = source.worker_request_id() { - #[cfg(target_os = "macos")] - { - let now = Instant::now(); - match last_block_reason { - Some("frame_matches_last_committed_frame") => self - .schedule_backoff_scroll_capture_worker_retry_if_fresh_downward_input( - now, - "worker_duplicate_committed_frame", - SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL, - ), - _ => self - .schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( - now, - "worker_no_change", - ), - } - } - tracing::info!( - op = "scroll_capture.worker_frame_processed", - request_id, - outcome = "no_change", - frame_px = ?frame_px, - input_direction = ?self.scroll_capture.input_direction, - last_block_reason = ?last_block_reason, - "Worker-fed scroll-capture frame reached the session without changing preview or export state." - ); - } + self.log_scroll_capture_no_change(source, frame_px) }, Ok(ScrollObserveOutcome::PreviewUpdated) => { - tracing::info!( - op = "scroll_capture.frame_observed", - frame_source = source.as_str(), - worker_request_id = ?source.worker_request_id(), - outcome = "preview_updated", - frame_px = ?frame_px, - input_direction = ?self.scroll_capture.input_direction, - input_gesture_active = self.scroll_capture.input_gesture_active, - export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), - preview_px = ?self.scroll_capture_preview_dimensions().map(|[w, h]| (w, h)), - "Scroll-capture observed a frame and advanced session sampling state without committing stitched growth." - ); - if let Some(request_id) = source.worker_request_id() { - tracing::info!( - op = "scroll_capture.worker_frame_processed", - request_id, - outcome = "preview_updated", - frame_px = ?frame_px, - input_direction = ?self.scroll_capture.input_direction, - "Worker-fed scroll-capture frame refreshed preview state without committing stitched growth." - ); - } + self.log_scroll_capture_preview_updated(source, frame_px); }, Ok(ScrollObserveOutcome::UnsupportedDirection { direction }) => { let export_size = self @@ -968,46 +982,7 @@ impl OverlaySession { ); }, Ok(ScrollObserveOutcome::Committed { direction, growth_rows }) => { - self.refresh_scroll_preview_committed_image(); - self.refresh_scroll_preview_display_image(); - self.sync_scroll_preview_segments(); - self.request_redraw_scroll_preview_window(); - - let telemetry = - self.scroll_capture.session.as_ref().map(ScrollSession::commit_telemetry); - let export_size = - telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.export_dimensions); - let preview_size = - telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.preview_dimensions); - - tracing::info!( - op = "scroll_capture.committed", - frame_source = source.as_str(), - worker_request_id = ?source.worker_request_id(), - direction = ?direction, - growth_rows = *growth_rows, - frame_px = ?frame_px, - export_px = ?export_size, - preview_px = ?preview_size, - current_viewport_top_y = ?telemetry.as_ref().map(|telemetry| telemetry.current_viewport_top_y), - growth_commit_count = ?telemetry.as_ref().map(|telemetry| telemetry.growth_commit_count), - preview_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.preview_segment_count), - export_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.export_segment_count), - last_commit_decision_source = - ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_decision_source), - last_commit_detected_motion_rows = - ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_detected_motion_rows), - last_commit_effective_motion_rows_hint = ?telemetry - .as_ref() - .map(|telemetry| telemetry.last_commit_effective_motion_rows_hint), - last_preview_segment_height_px = - ?telemetry.as_ref().map(|telemetry| telemetry.last_preview_segment_height_px), - last_export_segment_height_px = - ?telemetry.as_ref().map(|telemetry| telemetry.last_export_segment_height_px), - preview_export_segments_aligned = - ?telemetry.as_ref().map(|telemetry| telemetry.preview_export_segments_aligned), - "Scroll sample committed stitched growth." - ); + self.log_scroll_capture_committed(source, frame_px, *direction, *growth_rows); }, Err(err) => { self.scroll_capture_set_error(format!("{err:#}")); @@ -1015,6 +990,130 @@ impl OverlaySession { } } + fn log_scroll_capture_no_change( + &mut self, + source: ScrollCaptureFrameSource, + frame_px: (u32, u32), + ) { + let last_block_reason = + self.scroll_capture.session.as_ref().and_then(ScrollSession::last_block_reason); + + tracing::info!( + op = "scroll_capture.frame_observed", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + outcome = "no_change", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + input_gesture_active = self.scroll_capture.input_gesture_active, + last_block_reason = ?last_block_reason, + export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), + "Scroll-capture observed a frame but kept session state unchanged." + ); + + if let Some(request_id) = source.worker_request_id() { + #[cfg(target_os = "macos")] + { + let now = Instant::now(); + + match last_block_reason { + Some("frame_matches_last_committed_frame") => self + .schedule_backoff_scroll_capture_worker_retry_if_fresh_downward_input( + now, + "worker_duplicate_committed_frame", + SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL, + ), + _ => self + .schedule_immediate_scroll_capture_worker_retry_if_fresh_downward_input( + now, + "worker_no_change", + ), + } + } + + tracing::info!( + op = "scroll_capture.worker_frame_processed", + request_id, + outcome = "no_change", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + last_block_reason = ?last_block_reason, + "Worker-fed scroll-capture frame reached the session without changing preview or export state." + ); + } + } + + fn log_scroll_capture_preview_updated( + &self, + source: ScrollCaptureFrameSource, + frame_px: (u32, u32), + ) { + tracing::info!( + op = "scroll_capture.frame_observed", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + outcome = "preview_updated", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + input_gesture_active = self.scroll_capture.input_gesture_active, + export_px = ?self.scroll_capture.session.as_ref().map(ScrollSession::export_dimensions), + preview_px = ?self.scroll_capture_preview_dimensions().map(|[w, h]| (w, h)), + "Scroll-capture observed a frame and advanced session sampling state without committing stitched growth." + ); + + if let Some(request_id) = source.worker_request_id() { + tracing::info!( + op = "scroll_capture.worker_frame_processed", + request_id, + outcome = "preview_updated", + frame_px = ?frame_px, + input_direction = ?self.scroll_capture.input_direction, + "Worker-fed scroll-capture frame refreshed preview state without committing stitched growth." + ); + } + } + + fn log_scroll_capture_committed( + &mut self, + source: ScrollCaptureFrameSource, + frame_px: (u32, u32), + direction: ScrollDirection, + growth_rows: u32, + ) { + self.refresh_scroll_preview_committed_image(); + self.refresh_scroll_preview_display_image(); + self.sync_scroll_preview_segments(); + self.request_redraw_scroll_preview_window(); + + let telemetry = self.scroll_capture.session.as_ref().map(ScrollSession::commit_telemetry); + let export_size = + telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.export_dimensions); + let preview_size = + telemetry.as_ref().map_or((0, 0), |telemetry| telemetry.preview_dimensions); + + tracing::info!( + op = "scroll_capture.committed", + frame_source = source.as_str(), + worker_request_id = ?source.worker_request_id(), + direction = ?direction, + growth_rows, + frame_px = ?frame_px, + export_px = ?export_size, + preview_px = ?preview_size, + current_viewport_top_y = ?telemetry.as_ref().map(|telemetry| telemetry.current_viewport_top_y), + growth_commit_count = ?telemetry.as_ref().map(|telemetry| telemetry.growth_commit_count), + preview_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.preview_segment_count), + export_segment_count = ?telemetry.as_ref().map(|telemetry| telemetry.export_segment_count), + last_commit_decision_source = ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_decision_source), + last_commit_detected_motion_rows = ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_detected_motion_rows), + last_commit_effective_motion_rows_hint = ?telemetry.as_ref().map(|telemetry| telemetry.last_commit_effective_motion_rows_hint), + last_preview_segment_height_px = ?telemetry.as_ref().map(|telemetry| telemetry.last_preview_segment_height_px), + last_export_segment_height_px = ?telemetry.as_ref().map(|telemetry| telemetry.last_export_segment_height_px), + preview_export_segments_aligned = ?telemetry.as_ref().map(|telemetry| telemetry.preview_export_segments_aligned), + "Scroll sample committed stitched growth." + ); + } + pub(super) fn clear_scroll_capture_inflight_request(&mut self) { self.scroll_capture.inflight_request_id = None; #[cfg(target_os = "macos")] @@ -1053,6 +1152,7 @@ impl OverlaySession { let Some(observation) = self.scroll_capture.inflight_request_observation else { return false; }; + if !observation.was_observable || observation.external_input_seq == self.scroll_capture.last_external_scroll_input_seq { diff --git a/packages/rsnap-overlay/src/overlay/session_state.rs b/packages/rsnap-overlay/src/overlay/session_state.rs index 34a66ff3..02b2fdfa 100644 --- a/packages/rsnap-overlay/src/overlay/session_state.rs +++ b/packages/rsnap-overlay/src/overlay/session_state.rs @@ -1,8 +1,11 @@ +use std::collections::VecDeque; use std::{ collections::HashMap, time::{Duration, Instant}, }; +use image::RgbaImage; + use crate::overlay::{ DeviceCursorPointSource, FrozenToolbarTool, GlobalPoint, LIVE_PRESENT_INTERVAL_MIN, MonitorRect, PhysicalPosition, Pos2, REDRAW_SUBSTEP_CONTRIBUTION_FLOOR, RectPoints, @@ -195,7 +198,7 @@ pub(super) struct ScrollCaptureState { #[cfg(target_os = "macos")] pub(super) live_stream: Option, #[cfg(target_os = "macos")] - pub(super) live_stream_backlog: std::collections::VecDeque, + pub(super) live_stream_backlog: VecDeque, #[cfg(target_os = "macos")] pub(super) last_stream_frame_seq: u64, #[cfg(target_os = "macos")] @@ -222,10 +225,10 @@ pub(super) struct ScrollCaptureState { #[cfg(all(test, target_os = "macos"))] pub(super) force_worker_sampling_in_tests: bool, pub(super) session: Option, - pub(super) preview_committed_image: Option, - pub(super) preview_latest_frame: Option, - pub(super) preview_display_image: Option, - pub(super) retained_overlay_preview_image: Option, + pub(super) preview_committed_image: Option, + pub(super) preview_latest_frame: Option, + pub(super) preview_display_image: Option, + pub(super) retained_overlay_preview_image: Option, pub(super) retained_overlay_preview_motion_rows_hint: Option, pub(super) last_overlay_preview_motion_rows_hint: Option, pub(super) last_overlay_preview_provisional_motion_rows_hint: Option, @@ -248,7 +251,7 @@ pub(super) struct ScrollCaptureState { pub(super) struct ScrollCaptureLiveFrame { pub(super) frame_seq: u64, pub(super) captured_at: Instant, - pub(super) image: image::RgbaImage, + pub(super) image: RgbaImage, } #[cfg(target_os = "macos")] diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index 38460f88..8e775381 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -5,17 +5,16 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use color_eyre::eyre::{Result, WrapErr}; +use color_eyre::eyre::{self, Result, WrapErr}; use directories::ProjectDirs; use image::RgbaImage; use serde::{Deserialize, Serialize}; -use super::{MonitorRect, RectPoints, ScrollCaptureFrameSource}; +use crate::overlay::{MonitorRect, RectPoints, ScrollCaptureFrameSource}; +use crate::scroll_capture; use crate::{ png, - scroll_capture::{ - scroll_capture_fingerprint, ScrollDirection, ScrollObserveOutcome, ScrollSession, - }, + scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}, }; const SCROLL_CAPTURE_TRACE_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE"; @@ -23,7 +22,7 @@ const SCROLL_CAPTURE_TRACE_DIR_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE_DIR"; const SCROLL_CAPTURE_TRACE_SCHEMA: &str = "scroll_capture_live_trace/1"; const SCROLL_CAPTURE_TRACE_MANIFEST_FLUSH_INTERVAL: Duration = Duration::from_millis(250); -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureLiveTraceManifest { pub(crate) schema: String, pub(crate) trace_id: String, @@ -40,7 +39,7 @@ pub(crate) struct ScrollCaptureLiveTraceManifest { pub(crate) finalized: bool, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureTraceMonitor { pub(crate) id: u32, pub(crate) origin_x: i32, @@ -49,7 +48,6 @@ pub(crate) struct ScrollCaptureTraceMonitor { pub(crate) height: u32, pub(crate) scale_factor_x1000: u32, } - impl From for ScrollCaptureTraceMonitor { fn from(value: MonitorRect) -> Self { Self { @@ -63,28 +61,27 @@ impl From for ScrollCaptureTraceMonitor { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureTraceRect { pub(crate) x: u32, pub(crate) y: u32, pub(crate) width: u32, pub(crate) height: u32, } - impl From for ScrollCaptureTraceRect { fn from(value: RectPoints) -> Self { Self { x: value.x, y: value.y, width: value.width, height: value.height } } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(tag = "entry_type", rename_all = "snake_case")] pub(crate) enum ScrollCaptureLiveTraceEntry { Input(ScrollCaptureTraceInputEntry), Frame(ScrollCaptureTraceFrameEntry), } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureTraceInputEntry { pub(crate) applied_at_ms: u64, pub(crate) seq: u64, @@ -97,7 +94,7 @@ pub(crate) struct ScrollCaptureTraceInputEntry { pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureTraceFrameEntry { pub(crate) observed_at_ms: u64, pub(crate) allow_stale_input: bool, @@ -109,13 +106,12 @@ pub(crate) struct ScrollCaptureTraceFrameEntry { pub(crate) outcome: ScrollCaptureTraceRecordedOutcome, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub(crate) enum ScrollCaptureTraceFrameSource { Worker { request_id: u64 }, LiveStream { frame_seq: u64 }, } - impl From for ScrollCaptureTraceFrameSource { fn from(value: ScrollCaptureFrameSource) -> Self { match value { @@ -126,13 +122,12 @@ impl From for ScrollCaptureTraceFrameSource { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub(crate) enum ScrollCaptureTraceDirection { Up, Down, } - impl From for ScrollCaptureTraceDirection { fn from(value: ScrollDirection) -> Self { match value { @@ -142,7 +137,7 @@ impl From for ScrollCaptureTraceDirection { } } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub(crate) enum ScrollCaptureTraceRecordedOutcome { NoChange, @@ -151,7 +146,6 @@ pub(crate) enum ScrollCaptureTraceRecordedOutcome { Committed { direction: ScrollCaptureTraceDirection, growth_rows: u32 }, Error { message: String }, } - impl From for ScrollCaptureTraceRecordedOutcome { fn from(value: ScrollObserveOutcome) -> Self { match value { @@ -167,7 +161,7 @@ impl From for ScrollCaptureTraceRecordedOutcome { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ScrollCaptureTraceSessionSnapshot { pub(crate) input_direction: Option, pub(crate) input_gesture_active: bool, @@ -186,7 +180,6 @@ pub(crate) struct ScrollCaptureTraceSessionSnapshot { pub(crate) last_preview_segment_height_px: Option, pub(crate) last_export_segment_height_px: Option, } - impl ScrollCaptureTraceSessionSnapshot { pub(crate) fn capture( session: Option<&ScrollSession>, @@ -242,28 +235,6 @@ pub(crate) struct ScrollCaptureTraceRecorder { last_recorded_frame_path: Option, manifest: ScrollCaptureLiveTraceManifest, } - -pub(crate) struct ScrollCaptureTraceInputRecord { - pub(crate) seq: u64, - pub(crate) cursor_global: (f64, f64), - pub(crate) delta_y: f64, - pub(crate) gesture_active: bool, - pub(crate) gesture_ended: bool, - pub(crate) recorded_age: Duration, - pub(crate) applied_at: Instant, - pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, -} - -pub(crate) struct ScrollCaptureTraceFrameRecord<'a> { - pub(crate) frame: &'a RgbaImage, - pub(crate) source: ScrollCaptureFrameSource, - pub(crate) allow_stale_input: bool, - pub(crate) prior_block_reason: Option<&'static str>, - pub(crate) observed_at: Instant, - pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, - pub(crate) outcome: &'a Result, -} - impl ScrollCaptureTraceRecorder { pub(crate) fn from_env( monitor: MonitorRect, @@ -311,7 +282,7 @@ impl ScrollCaptureTraceRecorder { } pub(crate) fn record_frame_observation(&mut self, record: ScrollCaptureTraceFrameRecord<'_>) { - let frame_fingerprint = scroll_capture_fingerprint(record.frame); + let frame_fingerprint = scroll_capture::scroll_capture_fingerprint(record.frame); let frame_path = if self .last_recorded_frame_fingerprint .as_ref() @@ -319,12 +290,16 @@ impl ScrollCaptureTraceRecorder { { self.last_recorded_frame_path.clone().unwrap_or_else(|| { let frame_index = self.next_frame_index; + self.next_frame_index = self.next_frame_index.saturating_add(1); + format!("frames/frame-{frame_index:06}.png") }) } else { let frame_index = self.next_frame_index; + self.next_frame_index = self.next_frame_index.saturating_add(1); + let frame_path = format!("frames/frame-{frame_index:06}.png"); if let Err(err) = self.write_frame(record.frame, &frame_path) { @@ -335,12 +310,12 @@ impl ScrollCaptureTraceRecorder { "Failed to persist scroll-capture trace frame." ); } + self.last_recorded_frame_fingerprint = Some(frame_fingerprint); self.last_recorded_frame_path = Some(frame_path.clone()); frame_path }; - let outcome = match record.outcome { Ok(value) => ScrollCaptureTraceRecordedOutcome::from(*value), Err(err) => ScrollCaptureTraceRecordedOutcome::Error { message: format!("{err:#}") }, @@ -363,6 +338,7 @@ impl ScrollCaptureTraceRecorder { pub(crate) fn record_error(&mut self, message: &str) { self.manifest.final_error = Some(message.to_owned()); + self.flush_manifest_best_effort("record_error"); } @@ -385,7 +361,6 @@ impl ScrollCaptureTraceRecorder { } else { self.manifest.final_preview_path = Some(final_preview_path); } - if let Err(err) = self.write_frame(session.export_image(), &final_export_path) { tracing::warn!( op = "scroll_capture.trace_write_final_export_failed", @@ -398,6 +373,7 @@ impl ScrollCaptureTraceRecorder { } self.manifest.final_snapshot = Some(final_snapshot); + self.flush_manifest_best_effort("finalize_session"); } @@ -450,8 +426,11 @@ impl ScrollCaptureTraceRecorder { }; recorder.write_frame(base_frame, &recorder.manifest.base_frame_path)?; - recorder.last_recorded_frame_fingerprint = Some(scroll_capture_fingerprint(base_frame)); + + recorder.last_recorded_frame_fingerprint = + Some(scroll_capture::scroll_capture_fingerprint(base_frame)); recorder.last_recorded_frame_path = Some(recorder.manifest.base_frame_path.clone()); + recorder.flush_manifest_best_effort("init"); Ok(recorder) @@ -484,6 +463,7 @@ impl ScrollCaptureTraceRecorder { fn flush_manifest_if_due_best_effort(&mut self, op: &'static str) { let now = Instant::now(); + if now.saturating_duration_since(self.last_manifest_flush_at) < SCROLL_CAPTURE_TRACE_MANIFEST_FLUSH_INTERVAL { @@ -491,6 +471,7 @@ impl ScrollCaptureTraceRecorder { } self.last_manifest_flush_at = now; + self.flush_manifest_best_effort(op); } @@ -502,6 +483,7 @@ impl ScrollCaptureTraceRecorder { fs::write(&tmp_path, bytes).wrap_err_with(|| { format!("failed to write temporary trace manifest {}", tmp_path.display()) })?; + fs::rename(&tmp_path, &self.manifest_path).wrap_err_with(|| { format!( "failed to publish scroll-capture trace manifest {}", @@ -514,17 +496,38 @@ impl ScrollCaptureTraceRecorder { impl Drop for ScrollCaptureTraceRecorder { fn drop(&mut self) { self.manifest.finalized = true; + self.flush_manifest_best_effort("drop"); } } +pub(crate) struct ScrollCaptureTraceInputRecord { + pub(crate) seq: u64, + pub(crate) cursor_global: (f64, f64), + pub(crate) delta_y: f64, + pub(crate) gesture_active: bool, + pub(crate) gesture_ended: bool, + pub(crate) recorded_age: Duration, + pub(crate) applied_at: Instant, + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, +} + +pub(crate) struct ScrollCaptureTraceFrameRecord<'a> { + pub(crate) frame: &'a RgbaImage, + pub(crate) source: ScrollCaptureFrameSource, + pub(crate) allow_stale_input: bool, + pub(crate) prior_block_reason: Option<&'static str>, + pub(crate) observed_at: Instant, + pub(crate) snapshot_after: ScrollCaptureTraceSessionSnapshot, + pub(crate) outcome: &'a Result, +} + #[derive(Clone, Debug)] pub(crate) struct LoadedScrollCaptureLiveTrace { pub(crate) manifest_path: PathBuf, pub(crate) manifest: ScrollCaptureLiveTraceManifest, pub(crate) base_frame: RgbaImage, } - impl LoadedScrollCaptureLiveTrace { pub(crate) fn load(manifest_path: impl AsRef) -> Result { let manifest_path = manifest_path.as_ref().to_path_buf(); @@ -534,10 +537,7 @@ impl LoadedScrollCaptureLiveTrace { let manifest: ScrollCaptureLiveTraceManifest = serde_json::from_slice(&manifest_bytes) .wrap_err("failed to decode scroll-capture trace manifest")?; let base_dir = manifest_path.parent().ok_or_else(|| { - color_eyre::eyre::eyre!( - "trace manifest path {} has no parent directory", - manifest_path.display() - ) + eyre::eyre!("trace manifest path {} has no parent directory", manifest_path.display()) })?; let base_frame_path = base_dir.join(&manifest.base_frame_path); let base_frame = image::open(&base_frame_path) @@ -564,8 +564,10 @@ impl LoadedScrollCaptureLiveTrace { fn resolve_trace_root_dir() -> Option { let override_dir = env::var_os(SCROLL_CAPTURE_TRACE_DIR_ENV).and_then(|value| { let trimmed = value.to_string_lossy().trim().to_owned(); + if trimmed.is_empty() { None } else { Some(PathBuf::from(trimmed)) } }); + if let Some(dir) = override_dir { return Some(dir); } @@ -604,10 +606,19 @@ fn now_unix_ms() -> Result { #[cfg(test)] mod tests { + use std::process; use std::sync::atomic::{AtomicU64, Ordering}; - use super::*; use crate::GlobalPoint; + use crate::overlay::trace_recording; + use crate::overlay::trace_recording::SCROLL_CAPTURE_TRACE_SCHEMA; + use crate::overlay::trace_recording::{ + Duration, Instant, LoadedScrollCaptureLiveTrace, MonitorRect, PathBuf, RectPoints, + RgbaImage, ScrollCaptureLiveTraceEntry, ScrollCaptureTraceFrameRecord, + ScrollCaptureTraceInputRecord, ScrollCaptureTraceRecorder, + ScrollCaptureTraceSessionSnapshot, ScrollDirection, ScrollObserveOutcome, ScrollSession, + env, fs, + }; use crate::overlay::{OverlaySession, ScrollCaptureFrameSource}; static TRACE_TEST_ROOT_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -640,9 +651,9 @@ mod tests { fn temp_trace_root() -> PathBuf { let counter = TRACE_TEST_ROOT_COUNTER.fetch_add(1, Ordering::Relaxed); - let root = std::env::temp_dir().join(format!( + let root = env::temp_dir().join(format!( "rsnap-scroll-trace-test-{}-{}-{}", - now_unix_ms().unwrap_or(0), + trace_recording::now_unix_ms().unwrap_or(0), process::id(), counter )); @@ -664,6 +675,7 @@ mod tests { let base_frame = make_window(&rows, 0); let next_frame = make_window(&rows, 1); let root = temp_trace_root(); + let start = Instant::now(); let mut recorder = ScrollCaptureTraceRecorder::new_for_root_dir( root, test_monitor(), @@ -672,7 +684,6 @@ mod tests { &base_frame, ) .unwrap(); - let start = Instant::now(); let mut session = OverlaySession::new(); session.scroll_capture.active = true; @@ -723,6 +734,7 @@ mod tests { ), outcome: &Ok(ScrollObserveOutcome::PreviewUpdated), }); + let manifest_path = recorder.manifest_path().to_path_buf(); drop(recorder); @@ -762,6 +774,7 @@ mod tests { Some(&session), Some({ let image = session.preview_display_image(); + [image.width(), image.height()] }), Some(ScrollDirection::Down), @@ -769,9 +782,10 @@ mod tests { 0.0, Some(0), ); - let final_preview_image = session.preview_display_image(); + recorder.finalize_session(&session, &final_preview_image, final_snapshot); + drop(recorder); let loaded = LoadedScrollCaptureLiveTrace::load(&manifest_path).unwrap(); @@ -834,6 +848,7 @@ mod tests { snapshot_after: snapshot, outcome: &Ok(ScrollObserveOutcome::NoChange), }); + drop(recorder); let loaded = LoadedScrollCaptureLiveTrace::load(&manifest_path).unwrap(); diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 765f80a5..61ee6a1c 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -257,6 +257,7 @@ pub mod bench_support { } use std::ops::RangeInclusive; +use std::ptr; use color_eyre::eyre::{self, Result}; use image::{ @@ -267,6 +268,7 @@ use image::{ use objc2::{AnyThread, runtime::AnyObject}; #[cfg(target_os = "macos")] use objc2_core_foundation::CFData; +use objc2_core_foundation::CFRetained; #[cfg(target_os = "macos")] use objc2_core_graphics::{ CGBitmapInfo, CGColorRenderingIntent, CGColorSpace, CGDataProvider, CGImage, CGImageAlphaInfo, @@ -277,6 +279,10 @@ use objc2_foundation::{NSArray, NSDictionary}; #[cfg(target_os = "macos")] use objc2_vision::{VNImageOption, VNImageRequestHandler, VNTranslationalImageRegistrationRequest}; +pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 24; +pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS: u32 = 12; +pub(crate) const UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS: u32 = 8; + const FINGERPRINT_GRID_COLUMNS: u32 = 12; const FINGERPRINT_GRID_ROWS: u32 = 16; const DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 48; @@ -290,15 +296,12 @@ const DOWNWARD_REGISTRATION_MIN_OVERLAP_DIVISOR: u32 = 3; const DOWNWARD_KEYFRAME_SEARCH_LIMIT: usize = 4; const DOWNWARD_KEYFRAME_MIN_OVERLAP_DIVISOR: u32 = 5; const INITIAL_DOWNWARD_MAX_MOTION_ROWS: u32 = 256; -pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 24; -pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS: u32 = 12; const PREVIEW_ONLY_LOCAL_NEAR_CONTINUITY_ROWS: u32 = 4; const EXTREME_TRANSIENT_PREVIEW_LOCAL_TAIL_MULTIPLIER: u32 = 12; const REPEATED_PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS: u32 = 4; const TINY_OBSERVED_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 2; const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MAX_MOTION_ROWS: u32 = 1; const TINY_PREVIEW_ONLY_LOCAL_BURST_RECOVERY_MIN_LAST_HINT_ROWS: u32 = 7; -pub(crate) const UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS: u32 = 8; const BOOTSTRAP_HINTED_INITIAL_GROWTH_MAX_ROWS: u32 = 40; const DOWNWARD_COMMITTED_KEYFRAME_LOCAL_OVERRUN_MAX_ROWS: u32 = 24; const FALLBACK_DOWNWARD_GROWTH_MIN_ROWS: u32 = 8; @@ -448,21 +451,6 @@ impl Default for OverlapSearchConfig { } } -fn worker_pairwise_overlap_search_config() -> OverlapSearchConfig { - OverlapSearchConfig { - min_overlap_rows: 24, - max_column_samples: 96, - max_row_samples: 96, - max_mean_abs_diff_x100: 850, - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct PreviewOnlyDownwardLocalSample { - frame: RgbaImage, - viewport_top_y: i32, -} - #[derive(Clone, Debug)] pub(crate) struct ScrollSession { anchor_frame: RgbaImage, @@ -642,10 +630,7 @@ impl ScrollSession { let previous_worker_frame = self.worker_pairwise_previous_frame.clone(); if frame == previous_worker_frame { - self.record_last_sample(&frame, fingerprint.clone()); - self.record_last_downward_observed_sample(&frame, fingerprint); - self.worker_pairwise_previous_frame = frame; - self.clear_preview_only_downward_recovery_carryover(); + self.update_worker_pairwise_reference_frame(frame, fingerprint); self.log_decision( "scroll_capture.worker_pairwise_no_change", ScrollDirection::Down, @@ -661,10 +646,7 @@ impl ScrollSession { let Some(matched) = classify_vision_downward_sample_motion_against(&previous_worker_frame, &frame) else { - self.record_last_sample(&frame, fingerprint.clone()); - self.record_last_downward_observed_sample(&frame, fingerprint); - self.worker_pairwise_previous_frame = frame; - self.clear_preview_only_downward_recovery_carryover(); + self.update_worker_pairwise_reference_frame(frame, fingerprint); self.log_decision( "scroll_capture.worker_pairwise_no_change", ScrollDirection::Down, @@ -678,16 +660,14 @@ impl ScrollSession { }; let corroborated_shift_rows = estimate_pairwise_downward_shift_rows(&previous_worker_frame, &frame); + if matched.motion_rows >= WORKER_PAIRWISE_CORROBORATION_MIN_ROWS && corroborated_shift_rows.is_none_or(|estimated| { estimated == 0 || matched.motion_rows.abs_diff(estimated) > WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS }) { - self.record_last_sample(&frame, fingerprint.clone()); - self.record_last_downward_observed_sample(&frame, fingerprint); - self.worker_pairwise_previous_frame = frame; - self.clear_preview_only_downward_recovery_carryover(); + self.update_worker_pairwise_reference_frame(frame, fingerprint); self.log_decision( "scroll_capture.worker_pairwise_growth_blocked", ScrollDirection::Down, @@ -710,10 +690,7 @@ impl ScrollSession { let frame_max_growth_rows = frame.height().saturating_sub(1).max(1); if growth_rows == 0 || growth_rows > frame_max_growth_rows { - self.record_last_sample(&frame, fingerprint.clone()); - self.record_last_downward_observed_sample(&frame, fingerprint); - self.worker_pairwise_previous_frame = frame; - self.clear_preview_only_downward_recovery_carryover(); + self.update_worker_pairwise_reference_frame(frame, fingerprint); self.log_decision( "scroll_capture.worker_pairwise_growth_blocked", ScrollDirection::Down, @@ -740,7 +717,9 @@ impl ScrollSession { Some(growth_rows), Some("worker_pairwise_vision"), ); + self.worker_pairwise_previous_frame = frame.clone(); + self.clear_preview_only_downward_recovery_carryover(); self.apply_growth( @@ -754,6 +733,15 @@ impl ScrollSession { ) } + fn update_worker_pairwise_reference_frame(&mut self, frame: RgbaImage, fingerprint: Vec) { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + + self.worker_pairwise_previous_frame = frame; + + self.clear_preview_only_downward_recovery_carryover(); + } + fn observe_sample_with_motion_context( &mut self, frame: RgbaImage, @@ -763,12 +751,17 @@ impl ScrollSession { ) -> Result { let previous_hint = self.transient_motion_rows_hint; let previous_burst = self.transient_burst_search_enabled; + self.transient_motion_rows_hint = motion_rows_hint; self.transient_burst_search_enabled = allow_burst_search; + self.record_last_sample_eval_context(); + let result = self.observe_sample(frame, input_direction); + self.transient_motion_rows_hint = previous_hint; self.transient_burst_search_enabled = previous_burst; + result } @@ -786,6 +779,7 @@ impl ScrollSession { frame.width() )); } + let use_resume_local_sample = matches!(input_direction, ScrollDirection::Down) && self.resume_frontier_top_y.is_some(); @@ -798,6 +792,7 @@ impl ScrollSession { Some(0), Some("frame_matches_last_downward_observed_frame"), ); + return Ok(ScrollObserveOutcome::NoChange); } @@ -815,6 +810,7 @@ impl ScrollSession { Some(0), Some("frame_matches_last_unconfirmed_upward_fingerprint"), ); + return Ok(ScrollObserveOutcome::NoChange); } @@ -863,6 +859,7 @@ impl ScrollSession { Some(0), Some("sample_delta_and_motion_both_absent"), ); + return Ok(ScrollObserveOutcome::NoChange); } if matches!(input_direction, ScrollDirection::Up) { @@ -941,6 +938,7 @@ impl ScrollSession { Some(matched.source), Some(matched.matched.motion_rows), ); + ( Some(MotionObservation { direction: ScrollDirection::Down, @@ -971,6 +969,7 @@ impl ScrollSession { }, DownwardRegistrationWithSource::NoMatch => { self.record_last_downward_sample_registration("no_match", None, None); + (None, None) }, } @@ -1420,6 +1419,7 @@ impl ScrollSession { block_reason: Option<&'static str>, ) { self.last_block_reason = block_reason; + tracing::debug!( op, input_direction = ?input_direction, @@ -1581,6 +1581,7 @@ impl ScrollSession { fn clear_preview_only_downward_recovery_carryover(&mut self) { self.clear_preview_only_downward_local_sample(); + self.pending_suppressed_huge_preview_only_local_followup = None; self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 0; self.pending_extreme_preview_only_local_tail_followup = None; @@ -1639,7 +1640,9 @@ impl ScrollSession { frontier_top_y: i32, ) { self.last_motion_rows_hint = None; + self.clear_preview_only_downward_local_sample(); + self.resume_frontier_requires_reacquire = true; self.resume_frontier_top_y.get_or_insert(frontier_top_y); @@ -1649,6 +1652,7 @@ impl ScrollSession { fn observe_unconfirmed_upward_rewind(&mut self) { self.last_motion_rows_hint = None; + self.clear_preview_only_downward_local_sample(); let frontier_top_y = self.current_viewport_top_y; @@ -1660,6 +1664,7 @@ impl ScrollSession { self.observed_viewport_top_y.min(frontier_top_y.saturating_sub(1)); } + #[allow(clippy::too_many_lines)] fn observe_downward_motion( &mut self, frame: RgbaImage, @@ -1678,83 +1683,17 @@ impl ScrollSession { let candidate = match self.resolve_downward_viewport_candidate(&frame, observed_match) { DownwardViewportResolution::NoMatch => { - let reset_preview_only_local_baseline = - self.should_reset_preview_only_local_baseline_after_huge_far_committed_block(); - let preview_only_local_viewport_top_y = if self - .blocked_underconsumed_observed_recovery_in_burst - || self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst - || self.blocked_followup_after_suppressed_huge_preview_local_jump - || self.blocked_followup_after_extreme_preview_local_tail - || self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump - { - self.stable_preview_only_downward_local_viewport_top_y() - } else { - self.preview_only_downward_local_viewport_top_y_for_sample_match(observed_match) - }; - let block_reason = if self.blocked_underconsumed_observed_recovery_in_burst { - "underconsumed_observed_recovery_under_transient_burst" - } else if self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst { - "lagging_exactly_corroborated_preview_local_tail_under_transient_burst" - } else if self.blocked_followup_after_suppressed_huge_preview_local_jump { - "followup_after_suppressed_huge_preview_local_jump_under_transient_burst" - } else if self.blocked_followup_after_extreme_preview_local_tail { - "followup_after_extreme_preview_local_tail_under_transient_burst" - } else if self - .blocked_far_committed_only_recovery_after_corroborated_huge_local_jump - { - "far_committed_only_recovery_after_corroborated_huge_local_jump_under_transient_burst" - } else { - "no_downward_viewport_candidate_resolved" - }; - self.pending_unresolved_burst_registered_growth_viewport_top_y = if block_reason - == "no_downward_viewport_candidate_resolved" - && self.last_downward_sample_registration_result == Some("matched") - { - self.last_downward_sample_registration_provisional_viewport_top_y.filter( - |viewport_top_y| { - self.transient_burst_growth_matches_pending_hint_band(*viewport_top_y) - }, - ) - } else { - None - }; - self.consecutive_transient_burst_missing_downward_candidate_frames = if self - .transient_burst_search_enabled - && preview_only_local_viewport_top_y.is_some() - { - self.consecutive_transient_burst_missing_downward_candidate_frames - .saturating_add(1) - } else { - 0 - }; - self.refresh_local_downward_sample(&frame); - if self.should_refresh_downward_observed_baseline_after_huge_suppressed_jump() { - self.record_last_downward_observed_sample( - &frame, - scroll_capture_fingerprint(&frame), - ); - } - if reset_preview_only_local_baseline { - self.clear_preview_only_downward_local_sample(); - } else { - self.refresh_preview_only_downward_local_sample( - &frame, - preview_only_local_viewport_top_y, - ); - } - self.log_decision( - "scroll_capture.downward_viewport_authority_missing", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - None, - Some(0), - Some(block_reason), + return self.handle_missing_downward_viewport_authority( + &frame, + observed_match, + motion_rows, + preview_changed, ); - return Ok(preview_update_outcome(preview_changed)); }, DownwardViewportResolution::Selected(candidate) => candidate, DownwardViewportResolution::Ambiguous { preferred, competing } => { self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.refresh_local_downward_sample(&frame); self.refresh_preview_only_downward_local_sample( &frame, @@ -1779,112 +1718,65 @@ impl ScrollSession { }; if self.should_fail_closed_tiny_observed_recovery_in_burst(candidate) { - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("tiny_observed_recovery_under_transient_burst"), + motion_rows, + candidate, + preview_changed, + "tiny_observed_recovery_under_transient_burst", ); - return Ok(preview_update_outcome(preview_changed)); } if self.should_fail_closed_outsized_observed_recovery_after_one_pixel_preview_local_commit( candidate, ) { - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("outsized_observed_recovery_after_one_pixel_preview_local_commit"), + motion_rows, + candidate, + preview_changed, + "outsized_observed_recovery_after_one_pixel_preview_local_commit", ); - return Ok(preview_update_outcome(preview_changed)); } if self.should_fail_closed_tiny_preview_only_local_recovery_in_burst(candidate) { - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("tiny_preview_only_local_recovery_under_transient_burst"), + motion_rows, + candidate, + preview_changed, + "tiny_preview_only_local_recovery_under_transient_burst", ); - return Ok(preview_update_outcome(preview_changed)); } if self .should_fail_closed_exactly_corroborated_preview_local_tail_in_extreme_burst(candidate) { self.pending_extreme_preview_only_local_tail_followup = Some(candidate); self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 1; - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("exactly_corroborated_preview_local_tail_under_extreme_transient_burst"), + motion_rows, + candidate, + preview_changed, + "exactly_corroborated_preview_local_tail_under_extreme_transient_burst", ); - return Ok(preview_update_outcome(preview_changed)); } if self.should_fail_closed_preview_only_local_tail_after_unresolved_burst(candidate) { - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("preview_only_local_tail_after_unresolved_transient_burst"), + motion_rows, + candidate, + preview_changed, + "preview_only_local_tail_after_unresolved_transient_burst", ); - return Ok(preview_update_outcome(preview_changed)); } if self.should_fail_closed_tiny_committed_keyframe_recovery_in_burst(candidate) { - self.consecutive_transient_burst_missing_downward_candidate_frames = 0; - self.refresh_local_downward_sample(&frame); - self.refresh_preview_only_downward_local_sample( + return self.block_downward_growth_candidate( &frame, - self.stable_preview_only_downward_local_viewport_top_y(), - ); - self.log_decision( - "scroll_capture.downward_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate.viewport_top_y), - Some(candidate.motion_rows), - Some("tiny_committed_keyframe_recovery_under_transient_burst"), + motion_rows, + candidate, + preview_changed, + "tiny_committed_keyframe_recovery_under_transient_burst", ); - return Ok(preview_update_outcome(preview_changed)); } self.observe_downward_growth_to_viewport( @@ -1899,6 +1791,112 @@ impl ScrollSession { ) } + fn handle_missing_downward_viewport_authority( + &mut self, + frame: &RgbaImage, + observed_match: DownwardSampleMatch, + motion_rows: u32, + preview_changed: bool, + ) -> Result { + let reset_preview_only_local_baseline = + self.should_reset_preview_only_local_baseline_after_huge_far_committed_block(); + let preview_only_local_viewport_top_y = if self + .blocked_underconsumed_observed_recovery_in_burst + || self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst + || self.blocked_followup_after_suppressed_huge_preview_local_jump + || self.blocked_followup_after_extreme_preview_local_tail + || self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump + { + self.stable_preview_only_downward_local_viewport_top_y() + } else { + self.preview_only_downward_local_viewport_top_y_for_sample_match(observed_match) + }; + let block_reason = if self.blocked_underconsumed_observed_recovery_in_burst { + "underconsumed_observed_recovery_under_transient_burst" + } else if self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst { + "lagging_exactly_corroborated_preview_local_tail_under_transient_burst" + } else if self.blocked_followup_after_suppressed_huge_preview_local_jump { + "followup_after_suppressed_huge_preview_local_jump_under_transient_burst" + } else if self.blocked_followup_after_extreme_preview_local_tail { + "followup_after_extreme_preview_local_tail_under_transient_burst" + } else if self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump { + "far_committed_only_recovery_after_corroborated_huge_local_jump_under_transient_burst" + } else { + "no_downward_viewport_candidate_resolved" + }; + + self.pending_unresolved_burst_registered_growth_viewport_top_y = if block_reason + == "no_downward_viewport_candidate_resolved" + && self.last_downward_sample_registration_result == Some("matched") + { + self.last_downward_sample_registration_provisional_viewport_top_y.filter( + |viewport_top_y| { + self.transient_burst_growth_matches_pending_hint_band(*viewport_top_y) + }, + ) + } else { + None + }; + self.consecutive_transient_burst_missing_downward_candidate_frames = + if self.transient_burst_search_enabled && preview_only_local_viewport_top_y.is_some() { + self.consecutive_transient_burst_missing_downward_candidate_frames.saturating_add(1) + } else { + 0 + }; + + self.refresh_local_downward_sample(frame); + + if self.should_refresh_downward_observed_baseline_after_huge_suppressed_jump() { + self.record_last_downward_observed_sample(frame, scroll_capture_fingerprint(frame)); + } + if reset_preview_only_local_baseline { + self.clear_preview_only_downward_local_sample(); + } else { + self.refresh_preview_only_downward_local_sample( + frame, + preview_only_local_viewport_top_y, + ); + } + + self.log_decision( + "scroll_capture.downward_viewport_authority_missing", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + None, + Some(0), + Some(block_reason), + ); + + Ok(preview_update_outcome(preview_changed)) + } + + fn block_downward_growth_candidate( + &mut self, + frame: &RgbaImage, + motion_rows: u32, + candidate: DownwardViewportCandidate, + preview_changed: bool, + block_reason: &'static str, + ) -> Result { + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + + self.refresh_local_downward_sample(frame); + self.refresh_preview_only_downward_local_sample( + frame, + self.stable_preview_only_downward_local_viewport_top_y(), + ); + self.log_decision( + "scroll_capture.downward_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate.viewport_top_y), + Some(candidate.motion_rows), + Some(block_reason), + ); + + Ok(preview_update_outcome(preview_changed)) + } + fn should_fail_closed_tiny_observed_recovery_in_burst( &self, candidate: DownwardViewportCandidate, @@ -1934,6 +1932,7 @@ impl ScrollSession { if self.seeded_preview_only_local_catch_up_candidate_can_commit(candidate) { return false; } + let small_recovery_lags_recent_continuity = self.last_motion_rows_hint.is_some_and(|last_hint| { last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS @@ -1982,6 +1981,7 @@ impl ScrollSession { "CommittedKeyframe@{}/{}:", candidate.viewport_top_y, candidate.motion_rows ); + value.contains(&exact_committed) }) } @@ -2461,10 +2461,12 @@ impl ScrollSession { ) -> Result { let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); let effective_motion_rows_hint = self.effective_motion_rows_hint(); + self.pending_unresolved_burst_registered_growth_viewport_top_y = None; if self.bootstrap_initial_growth_cap_rows().is_some_and(|cap| growth_rows > cap) { self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.log_decision( "scroll_capture.downward_growth_blocked", ScrollDirection::Down, @@ -2476,9 +2478,9 @@ impl ScrollSession { return Ok(preview_update_outcome(preview_changed)); } - if growth_rows == 0 { self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + let block_reason = if self.resume_frontier_top_y.is_some() { Some("candidate_viewport_did_not_pass_resume_frontier") } else { @@ -2496,10 +2498,12 @@ impl ScrollSession { return Ok(preview_update_outcome(preview_changed)); } + let max_growth_rows = self.max_downward_growth_rows_for_frame(&frame); if growth_rows > max_growth_rows { self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + self.log_decision( "scroll_capture.downward_growth_blocked", ScrollDirection::Down, @@ -2511,6 +2515,7 @@ impl ScrollSession { return Ok(preview_update_outcome(preview_changed)); } + self.log_decision( "scroll_capture.downward_growth_candidate", ScrollDirection::Down, @@ -2519,8 +2524,11 @@ impl ScrollSession { Some(growth_rows), Some(decision_source), ); + self.consecutive_transient_burst_missing_downward_candidate_frames = 0; + let previous_motion_rows_hint = self.last_motion_rows_hint; + self.last_motion_rows_hint = Some(growth_rows); self.apply_growth( @@ -2621,18 +2629,20 @@ impl ScrollSession { ) -> DownwardRegistrationWithSource { let (primary_raw, primary_reason) = self.classify_downward_sample_motion(frame); let primary = primary_raw.map_source(DownwardSampleMatchSource::ObservedSample); + self.record_registration_diagnostics( DownwardSampleMatchSource::ObservedSample, primary, primary_reason, ); + let Some(previous_local) = self.last_preview_only_downward_local_sample.as_ref() else { return primary; }; - let (local_raw, local_reason) = self.classify_preview_only_local_recovery_motion_against(&previous_local.frame, frame); let local = local_raw.map_source(DownwardSampleMatchSource::PreviewOnlyLocalSample); + self.record_registration_diagnostics( DownwardSampleMatchSource::PreviewOnlyLocalSample, local, @@ -2644,19 +2654,18 @@ impl ScrollSession { DownwardRegistrationWithSource::Matched(primary), DownwardRegistrationWithSource::Matched(local), ) => { - if self.should_prefer_preview_only_local_recovery_after_extreme_tail_block( - primary, local, - ) { - DownwardRegistrationWithSource::Matched(local) - } else if self - .should_prefer_observed_sample_over_preview_only_local_recovery(primary, local) - { - DownwardRegistrationWithSource::Matched(primary) - } else if self - .should_prefer_preview_only_local_recovery_over_observed_sample(primary, local) - { - DownwardRegistrationWithSource::Matched(local) - } else if local.matched.mean_abs_diff_x100 <= primary.matched.mean_abs_diff_x100 { + let prefer_local = + self.should_prefer_preview_only_local_recovery_after_extreme_tail_block( + primary, local, + ) || (!self.should_prefer_observed_sample_over_preview_only_local_recovery( + primary, local, + ) && (self + .should_prefer_preview_only_local_recovery_over_observed_sample( + primary, local, + ) || local.matched.mean_abs_diff_x100 + <= primary.matched.mean_abs_diff_x100)); + + if prefer_local { DownwardRegistrationWithSource::Matched(local) } else { DownwardRegistrationWithSource::Matched(primary) @@ -2827,7 +2836,6 @@ impl ScrollSession { ) -> (DownwardRegistration, Option<&'static str>) { let config = OverlapSearchConfig::default(); let preferred_ranges = self.sequential_downward_motion_ranges(previous, frame, config); - let (registration, reason) = self .evaluate_reference_downward_registration_with_preferred_ranges( previous, @@ -2858,7 +2866,6 @@ impl ScrollSession { let preferred_ranges = preferred_range.into_iter().collect::>(); let motion_rows_hint = self.last_motion_rows_hint.or(self.normalized_transient_motion_rows_hint()); - let (registration, reason) = self .evaluate_reference_downward_registration_with_preferred_ranges( previous, @@ -2891,6 +2898,7 @@ impl ScrollSession { fn normalized_transient_motion_rows_hint(&self) -> Option { let transient = self.transient_motion_rows_hint?; + if self.transient_burst_search_enabled { return Some(transient); } @@ -2940,7 +2948,6 @@ impl ScrollSession { if max_motion_rows == 0 { return None; } - if self.initial_downward_bootstrap_active() && self.last_motion_rows_hint.is_none() { if let Some(hint) = self.normalized_transient_motion_rows_hint() && hint <= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS @@ -3005,7 +3012,9 @@ impl ScrollSession { } self.last_unconfirmed_upward_fingerprint = None; + let fingerprint = scroll_capture_fingerprint(frame); + self.record_last_sample(frame, fingerprint); } @@ -3016,12 +3025,16 @@ impl ScrollSession { ) { let Some(provisional_viewport_top_y) = provisional_viewport_top_y else { self.clear_preview_only_downward_local_sample(); + return; }; + if !self.should_refresh_preview_only_downward_local_sample(frame) { return; } + self.last_unconfirmed_upward_fingerprint = None; + self.record_preview_only_downward_local_sample(frame, provisional_viewport_top_y); } @@ -3254,7 +3267,9 @@ impl ScrollSession { self.last_downward_observed_frame = previous.frame.clone(); self.last_downward_observed_fingerprint = Some(scroll_capture_fingerprint(&previous.frame)); + self.clear_preview_only_downward_local_sample(); + self.last_unconfirmed_upward_fingerprint = None; self.resume_frontier_top_y = None; self.resume_frontier_requires_reacquire = false; @@ -3266,7 +3281,9 @@ impl ScrollSession { self.last_downward_observed_frame = self.anchor_frame.clone(); self.last_downward_observed_fingerprint = Some(scroll_capture_fingerprint(&self.anchor_frame)); + self.clear_preview_only_downward_local_sample(); + self.last_unconfirmed_upward_fingerprint = None; self.last_motion_rows_hint = None; self.current_viewport_top_y = 0; @@ -3349,9 +3366,11 @@ impl ScrollSession { ); no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; } + candidates.retain(|matched| { downward_registration_has_meaningful_overlap(*matched, max_overlap, config) }); + if candidates.is_empty() { no_match_reason.get_or_insert("insufficient_overlap"); } @@ -3411,9 +3430,11 @@ impl ScrollSession { ); no_match_reason = if candidates.is_empty() { Some("no_candidates") } else { None }; } + candidates.retain(|matched| { downward_registration_has_meaningful_overlap(*matched, max_overlap, config) }); + if candidates.is_empty() { no_match_reason.get_or_insert("insufficient_overlap"); } @@ -3435,6 +3456,7 @@ impl ScrollSession { }, (DownwardRegistration::NoMatch, _) => { let _ = no_match_reason; + DownwardRegistration::NoMatch }, (other, _) => other, @@ -3447,8 +3469,9 @@ impl ScrollSession { next: &RgbaImage, config: OverlapSearchConfig, ) -> Vec> { - let mut ranges = Vec::new(); let local_motion_rows_hint = self.last_motion_rows_hint; + let mut ranges = Vec::new(); + if let Some(local_range) = self.preferred_local_downward_motion_range_from_hint( previous, next, @@ -3457,9 +3480,11 @@ impl ScrollSession { ) { ranges.push(local_range); } + if self.initial_downward_bootstrap_active() && self.last_motion_rows_hint.is_none() { return ranges; } + if let Some(transient_range) = self.transient_downward_motion_range(previous, next, config) && !ranges.contains(&transient_range) { @@ -3682,18 +3707,22 @@ impl ScrollSession { self.pending_suppressed_huge_preview_only_local_followup.take(); let pending_suppressed_huge_preview_only_local_followup_remaining_blocks = self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks; + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = 0; + let pending_extreme_preview_only_local_tail_followup = self.pending_extreme_preview_only_local_tail_followup.take(); let pending_extreme_preview_only_local_tail_followup_remaining_blocks = self.pending_extreme_preview_only_local_tail_followup_remaining_blocks; + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = 0; + + let provisional_viewport_top_y = + self.provisional_viewport_top_y_for_downward_sample_match(observed_match); let mut candidates = Vec::with_capacity(DOWNWARD_KEYFRAME_SEARCH_LIMIT.saturating_add(1)); let mut suppressed_observed_candidate = None; let mut suppressed_preview_only_local_candidate = None; - let provisional_viewport_top_y = - self.provisional_viewport_top_y_for_downward_sample_match(observed_match); self.last_downward_sample_registration_provisional_viewport_top_y = provisional_viewport_top_y; @@ -3720,69 +3749,41 @@ impl ScrollSession { suppressed_preview_only_local_candidate = Some(candidate); } } + self.collect_committed_downward_viewport_candidates(frame, &mut candidates); - if self - .should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( - suppressed_preview_only_local_candidate, - &candidates, - ) { - self.pending_suppressed_huge_preview_only_local_followup = - suppressed_preview_only_local_candidate; - self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = self - .suppressed_huge_preview_only_local_followup_block_budget( - suppressed_preview_only_local_candidate, - ); - candidates.clear(); - } - if self.should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + self.apply_pending_preview_local_followup_blocks( + suppressed_preview_only_local_candidate, pending_suppressed_huge_preview_only_local_followup, - &candidates, - ) { - if let Some(pending_candidate) = pending_suppressed_huge_preview_only_local_followup { - if pending_suppressed_huge_preview_only_local_followup_remaining_blocks > 1 { - self.pending_suppressed_huge_preview_only_local_followup = - Some(pending_candidate); - self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = - pending_suppressed_huge_preview_only_local_followup_remaining_blocks - 1; - } - } - self.blocked_followup_after_suppressed_huge_preview_local_jump = true; - candidates.clear(); - } - if self.should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + pending_suppressed_huge_preview_only_local_followup_remaining_blocks, pending_extreme_preview_only_local_tail_followup, - &candidates, - ) { - if let Some(pending_candidate) = pending_extreme_preview_only_local_tail_followup - && pending_extreme_preview_only_local_tail_followup_remaining_blocks > 1 - { - self.pending_extreme_preview_only_local_tail_followup = Some(pending_candidate); - self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = - pending_extreme_preview_only_local_tail_followup_remaining_blocks - 1; - } - self.blocked_followup_after_extreme_preview_local_tail = true; - candidates.clear(); - } + pending_extreme_preview_only_local_tail_followup_remaining_blocks, + &mut candidates, + ); self.restore_corroborated_observed_candidate( suppressed_observed_candidate, &mut candidates, ); + let preview_only_local_candidate_before_prune = candidates.iter().copied().find(|candidate| { candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample }); let candidates_before_prune = candidates.clone(); + self.last_downward_viewport_candidates_before_prune = Some(format_downward_viewport_candidates(&candidates)); + self.prune_committed_keyframe_candidates_outside_local_continuity(&mut candidates); self.restore_repeated_small_preview_only_local_candidate_after_empty_prune( preview_only_local_candidate_before_prune, &mut candidates, ); + if self.should_fail_closed_lagging_exactly_corroborated_preview_local_tail_in_burst( &candidates, ) { self.blocked_lagging_exactly_corroborated_preview_local_tail_in_burst = true; + candidates.clear(); } if self.should_fail_closed_underconsumed_observed_recovery_in_burst( @@ -3790,14 +3791,75 @@ impl ScrollSession { &candidates, ) { self.blocked_underconsumed_observed_recovery_in_burst = true; + candidates.clear(); } + self.last_downward_viewport_candidate_count = Some(candidates.len()); self.last_downward_viewport_candidates_after_prune = Some(format_downward_viewport_candidates(&candidates)); + select_downward_viewport_candidate(&mut candidates) } + #[allow(clippy::too_many_arguments)] + fn apply_pending_preview_local_followup_blocks( + &mut self, + suppressed_preview_only_local_candidate: Option, + pending_suppressed_huge_preview_only_local_followup: Option, + pending_suppressed_huge_preview_only_local_followup_remaining_blocks: u8, + pending_extreme_preview_only_local_tail_followup: Option, + pending_extreme_preview_only_local_tail_followup_remaining_blocks: u8, + candidates: &mut Vec, + ) { + if self + .should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( + suppressed_preview_only_local_candidate, + candidates, + ) { + self.pending_suppressed_huge_preview_only_local_followup = + suppressed_preview_only_local_candidate; + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = self + .suppressed_huge_preview_only_local_followup_block_budget( + suppressed_preview_only_local_candidate, + ); + + candidates.clear(); + } + if self.should_fail_closed_committed_followup_after_suppressed_huge_preview_local_jump( + pending_suppressed_huge_preview_only_local_followup, + candidates, + ) { + if let Some(pending_candidate) = pending_suppressed_huge_preview_only_local_followup + && pending_suppressed_huge_preview_only_local_followup_remaining_blocks > 1 + { + self.pending_suppressed_huge_preview_only_local_followup = Some(pending_candidate); + self.pending_suppressed_huge_preview_only_local_followup_remaining_blocks = + pending_suppressed_huge_preview_only_local_followup_remaining_blocks - 1; + } + + self.blocked_followup_after_suppressed_huge_preview_local_jump = true; + + candidates.clear(); + } + if self.should_fail_closed_committed_followup_after_extreme_preview_local_tail_block( + pending_extreme_preview_only_local_tail_followup, + candidates, + ) { + if let Some(pending_candidate) = pending_extreme_preview_only_local_tail_followup + && pending_extreme_preview_only_local_tail_followup_remaining_blocks > 1 + { + self.pending_extreme_preview_only_local_tail_followup = Some(pending_candidate); + self.pending_extreme_preview_only_local_tail_followup_remaining_blocks = + pending_extreme_preview_only_local_tail_followup_remaining_blocks - 1; + } + + self.blocked_followup_after_extreme_preview_local_tail = true; + + candidates.clear(); + } + } + fn should_fail_closed_suppressed_huge_preview_local_jump_corroborated_by_observed_and_committed( &self, suppressed_preview_only_local_candidate: Option, @@ -3809,6 +3871,7 @@ impl ScrollSession { let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { return false; } @@ -3842,6 +3905,7 @@ impl ScrollSession { let Some(pending_candidate) = pending_suppressed_preview_only_local_candidate else { return false; }; + if pending_candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { return false; } @@ -3867,6 +3931,7 @@ impl ScrollSession { let Some(pending_candidate) = pending_preview_only_local_candidate else { return false; }; + if pending_candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { return false; } @@ -3890,6 +3955,7 @@ impl ScrollSession { let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return 3; }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample { return 3; } @@ -3910,6 +3976,7 @@ impl ScrollSession { let Some(candidate) = suppressed_observed_candidate else { return; }; + if !self.observed_candidate_can_recover_from_committed_corroboration(candidate) { return; } @@ -3946,13 +4013,16 @@ impl ScrollSession { ) { let Some(candidate) = preview_only_local_candidate_before_prune else { self.last_blocked_preview_only_local_candidate = None; + return; }; + if candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample || !candidates_after_prune.is_empty() || !self.repeated_preview_only_local_candidate_can_restore_after_empty_prune(candidate) { self.last_blocked_preview_only_local_candidate = None; + return; } @@ -3960,11 +4030,13 @@ impl ScrollSession { Some(previous) if previous.candidate == candidate => previous.repeats.saturating_add(1), _ => 1, }; + self.last_blocked_preview_only_local_candidate = Some(BlockedPreviewOnlyLocalCandidate { candidate, repeats }); if repeats >= 2 { candidates_after_prune.push(candidate); + self.last_blocked_preview_only_local_candidate = None; } } @@ -3986,6 +4058,7 @@ impl ScrollSession { if !self.transient_burst_search_enabled { return false; } + let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; @@ -4025,7 +4098,6 @@ impl ScrollSession { else { return false; }; - let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; @@ -4047,6 +4119,7 @@ impl ScrollSession { && candidate.viewport_top_y == observed_candidate.viewport_top_y && candidate.motion_rows == observed_candidate.motion_rows }); + if !has_same_motion_committed_corroboration { return false; } @@ -4073,6 +4146,7 @@ impl ScrollSession { candidate.source == DownwardViewportCandidateSource::CommittedKeyframe }); let mut local_anchor = best_local_downward_viewport_candidate(candidates); + if local_anchor.is_some_and(|anchor| { has_committed_candidate && anchor.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample @@ -4092,6 +4166,7 @@ impl ScrollSession { candidates.retain(|candidate| { candidate.source != DownwardViewportCandidateSource::PreviewOnlyLocalSample }); + local_anchor = best_local_downward_viewport_candidate(candidates); } @@ -4115,7 +4190,9 @@ impl ScrollSession { <= max_bootstrap_growth_rows }); } + self.prune_committed_keyframe_candidates_without_local_anchor(candidates); + return; }; let allowed_overrun_rows = self @@ -4182,6 +4259,7 @@ impl ScrollSession { }) else { return; }; + if self.should_fail_closed_far_committed_only_recovery_without_local_anchor( preferred, candidates, ) { @@ -4192,7 +4270,9 @@ impl ScrollSession { ) { self.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; } + candidates.clear(); + return; } @@ -4207,11 +4287,14 @@ impl ScrollSession { let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; + if !self.transient_burst_search_enabled { return false; } + let preferred_growth_rows = self.growth_rows_for_candidate_viewport_top_y(preferred.viewport_top_y); + if self .should_fail_closed_underconsumed_committed_only_recovery_after_suppressed_preview_local_match( preferred, @@ -4282,6 +4365,7 @@ impl ScrollSession { let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; + if preferred.source != DownwardViewportCandidateSource::CommittedKeyframe { return false; } @@ -4357,7 +4441,6 @@ impl ScrollSession { let Some(local_motion_rows) = self.last_preview_only_local_registration_motion_rows else { return false; }; - let corroborated_motion_floor = last_motion_rows_hint.saturating_add(DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS); let corroborated_motion_ceiling = observed_motion_rows @@ -4388,7 +4471,6 @@ impl ScrollSession { let Some(observed_motion_rows) = self.last_observed_sample_registration_motion_rows else { return false; }; - let recent_preview_local_commit = self.growth_history.last().is_some_and(|commit| { commit.decision_source == DownwardViewportCandidateSource::PreviewOnlyLocalSample.decision_source() @@ -4439,6 +4521,7 @@ impl ScrollSession { } let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + self.transient_burst_motion_hint_exceeds_local_authority(candidate.motion_rows) && self.last_motion_rows_hint.is_some_and(|last_hint| { candidate.motion_rows.saturating_add(UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS) @@ -4458,6 +4541,7 @@ impl ScrollSession { candidate.source == DownwardViewportCandidateSource::PreviewOnlyLocalSample && { let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate.viewport_top_y); + growth_rows >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS || self.last_motion_rows_hint.is_some_and(|last_hint| { last_hint >= PREVIEW_ONLY_LOCAL_RECOVERY_MAX_MOTION_ROWS @@ -4519,7 +4603,6 @@ impl ScrollSession { else { return; }; - let Some(previous_growth_rows) = self.last_motion_rows_hint else { return; }; @@ -4601,9 +4684,11 @@ impl ScrollSession { let Some(transient_growth_cap_rows) = self.transient_pending_growth_cap_rows() else { return false; }; + if committed_candidate.source != DownwardViewportCandidateSource::CommittedKeyframe { return false; } + let local_growth_rows = self.growth_rows_for_candidate_viewport_top_y(local_anchor.viewport_top_y); let committed_growth_rows = @@ -4757,6 +4842,7 @@ impl ScrollSession { None, true, ); + registration = self.prefer_full_range_committed_keyframe_registration( registration, full_range_registration, @@ -4805,7 +4891,6 @@ impl ScrollSession { let Some(last_motion_rows_hint) = self.last_motion_rows_hint else { return false; }; - let low_confidence_match = matched.mean_abs_diff_x100 > DIRECTION_WARNING_MARGIN_X100.saturating_mul(4); let tiny_underconsumed_match = self @@ -4991,6 +5076,7 @@ impl ScrollSession { } } + #[allow(clippy::too_many_arguments)] fn apply_growth( &mut self, frame: RgbaImage, @@ -5014,14 +5100,17 @@ impl ScrollSession { self.current_viewport_top_y = viewport_top_y; self.observed_viewport_top_y = viewport_top_y; + self.record_last_sample(&frame, fingerprint); self.record_last_downward_observed_sample(&frame, scroll_capture_fingerprint(&frame)); + if self.should_seed_preview_only_local_after_observed_burst_commit( decision_source, growth_rows, previous_motion_rows_hint, ) { self.record_preview_only_downward_local_sample(&frame, viewport_top_y); + self.seeded_preview_only_local_after_observed_burst_commit = true; } else if self.should_preserve_preview_only_local_after_preview_only_burst_commit( decision_source, @@ -5029,11 +5118,13 @@ impl ScrollSession { previous_motion_rows_hint, ) { self.record_preview_only_downward_local_sample(&frame, viewport_top_y); + self.seeded_preview_only_local_after_observed_burst_commit = false; self.last_blocked_preview_only_local_candidate = None; } else { self.clear_preview_only_downward_local_sample(); } + self.last_unconfirmed_upward_fingerprint = None; self.last_committed_frame = frame.clone(); self.resume_frontier_top_y = None; @@ -5112,165 +5203,167 @@ impl ScrollSession { } } +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreviewOnlyDownwardLocalSample { + frame: RgbaImage, + viewport_top_y: i32, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct DirectionMatch { mean_abs_diff_x100: u32, motion_rows: u32, } -#[cfg(target_os = "macos")] -fn classify_vision_downward_sample_motion_against( - previous: &RgbaImage, - next: &RgbaImage, -) -> Option { - let previous_cg = cg_image_from_rgba_image(previous).ok()?; - let next_cg = cg_image_from_rgba_image(next).ok()?; - let options = NSDictionary::::new(); - let request = unsafe { - VNTranslationalImageRegistrationRequest::initWithTargetedCGImage_options( - VNTranslationalImageRegistrationRequest::alloc(), - previous_cg.as_ref(), - options.as_ref(), - ) - }; - let request_array = NSArray::from_retained_slice(&[request - .clone() - .into_super() - .into_super() - .into_super() - .into_super()]); - let handler = unsafe { - VNImageRequestHandler::initWithCGImage_options( - VNImageRequestHandler::alloc(), - next_cg.as_ref(), - options.as_ref(), - ) - }; - - handler.performRequests_error(request_array.as_ref()).ok()?; - - let results = unsafe { request.results() }?; - if results.count() == 0 { - return None; - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DownwardSampleMatch { + matched: DirectionMatch, + source: DownwardSampleMatchSource, +} - let translation = unsafe { results.objectAtIndex(0).alignmentTransform() }; - let motion_rows = translation.ty.round(); - if !motion_rows.is_finite() || motion_rows <= 0.0 { - return None; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DownwardViewportCandidate { + source: DownwardViewportCandidateSource, + viewport_top_y: i32, + motion_rows: u32, + mean_abs_diff_x100: u32, +} +impl DownwardViewportCandidate { + fn competing_block_reason(self, competing: Self) -> &'static str { + match (self.source, competing.source) { + ( + DownwardViewportCandidateSource::CommittedKeyframe, + DownwardViewportCandidateSource::CommittedKeyframe, + ) => "conflicting_committed_keyframe_authority", + _ => "conflicting_downward_viewport_authority", + } } - let motion_rows = motion_rows as u32; - let config = OverlapSearchConfig::default(); - let matched = evaluate_overlap_direction( - previous, - next, - ScrollDirection::Down, - motion_rows..=motion_rows, - config, - )?; - let max_overlap = previous.height().min(next.height()); - - downward_registration_has_meaningful_overlap(matched, max_overlap, config).then_some(matched) } -#[cfg(not(target_os = "macos"))] -fn classify_vision_downward_sample_motion_against( - _previous: &RgbaImage, - _next: &RgbaImage, -) -> Option { - None +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct BlockedPreviewOnlyLocalCandidate { + candidate: DownwardViewportCandidate, + repeats: u8, } -fn estimate_pairwise_downward_shift_rows(previous: &RgbaImage, current: &RgbaImage) -> Option { - if previous.dimensions() != current.dimensions() { - return None; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct OverlapSearchRange { + start: u32, + end: u32, +} +impl OverlapSearchRange { + fn as_range(self) -> RangeInclusive { + self.start..=self.end } - let (_width, height) = previous.dimensions(); - if height < 3 { - return None; +} + +impl From> for OverlapSearchRange { + fn from(range: RangeInclusive) -> Self { + Self { start: *range.start(), end: *range.end() } } - let max_shift = height.saturating_sub(1); +} - evaluate_overlap_direction( - previous, - current, - ScrollDirection::Down, - 1..=max_shift, - worker_pairwise_overlap_search_config(), - ) - .map(|matched| matched.motion_rows) +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DirectionMatchEval { + preferred_range: Option, + max_motion_rows: u32, + preferred_only_match: Option, + final_match: Option, + used_full_range_fallback: bool, } -#[cfg(target_os = "macos")] -fn cg_image_from_rgba_image( - image: &RgbaImage, -) -> Result> { - let width = image.width() as usize; - let height = image.height() as usize; - if width == 0 || height == 0 { - return Err(eyre::eyre!("vision registration image has zero dimensions")); - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct MotionObservation { + direction: ScrollDirection, + motion_rows: u32, +} - let bytes = CFData::from_bytes(image.as_raw()); - let provider = CGDataProvider::with_cf_data(Some(bytes.as_ref())) - .ok_or_else(|| eyre::eyre!("failed to create CGDataProvider for Vision registration"))?; - let color_space = CGColorSpace::new_device_rgb() - .ok_or_else(|| eyre::eyre!("failed to create RGB colorspace for Vision registration"))?; - let bitmap_info = CGBitmapInfo(CGImageAlphaInfo::Last.0 | CGImageByteOrderInfo::Order32Big.0); +#[derive(Clone, Copy, Debug)] +struct UpInputMatchLog { + sample_motion: Option, + sample_down_match: Option, + sample_up_match: Option, + committed_down_match: Option, + committed_up_match: Option, + sample_override_wins: bool, + committed_override_wins: bool, +} - unsafe { - CGImage::new( - width, - height, - 8, - 32, - width.saturating_mul(4), - Some(color_space.as_ref()), - bitmap_info, - Some(provider.as_ref()), - std::ptr::null(), - false, - CGColorRenderingIntent::RenderingIntentDefault, - ) - } - .ok_or_else(|| eyre::eyre!("failed to create CGImage for Vision registration")) +#[derive(Clone, Copy, Debug)] +struct UpInputSearchWindowLog<'a> { + sample_delta: Option, + sample_down_match_eval: &'a DirectionMatchEval, + sample_up_match_eval: &'a DirectionMatchEval, + committed_down_match_eval: &'a DirectionMatchEval, + committed_up_match_eval: &'a DirectionMatchEval, + frame_equals_last_sample: bool, + frame_equals_last_committed: bool, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardRegistration { - NoMatch, - Matched(DirectionMatch), - Ambiguous { best: DirectionMatch, competing: DirectionMatch }, +#[derive(Clone, Copy, Debug)] +struct UpwardInputDiagnostics { + sample_down_match_eval: DirectionMatchEval, + sample_up_match_eval: DirectionMatchEval, + committed_down_match_eval: DirectionMatchEval, + committed_up_match_eval: DirectionMatchEval, + sample_override_match: Option, + committed_override_match: Option, +} + +#[derive(Clone, Copy, Debug)] +struct ResumeFrontierMatchLog { + motion_rows: u32, + candidate_observed_viewport_top_y: i32, + residual_growth_rows: u32, + raw_committed_down_match: Option, + trusted_committed_down_match: Option, + committed_up_match: Option, + frame_reacquires_last_committed_viewport: bool, +} + +#[derive(Clone, Copy, Debug)] +struct ResumeFrontierDirectMatchContext { + motion_rows: u32, + candidate_observed_viewport_top_y: i32, + residual_growth_rows: u32, +} + +#[derive(Clone, Debug)] +struct GrowthCommit { + frame: RgbaImage, + growth_rows: u32, + viewport_top_y: i32, + decision_source: &'static str, + detected_motion_rows: Option, + effective_motion_rows_hint: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardSampleMatchSource { - ObservedSample, - PreviewOnlyLocalSample, +struct InformativeSpan { + start_x: u32, + end_exclusive_x: u32, } -impl DownwardSampleMatchSource { - const fn label(self) -> &'static str { - match self { - Self::ObservedSample => "observed_sample", - Self::PreviewOnlyLocalSample => "preview_only_local_sample", - } - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ScrollDirection { + Up, + Down, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DownwardSampleMatch { - matched: DirectionMatch, - source: DownwardSampleMatchSource, +pub(crate) enum ScrollObserveOutcome { + NoChange, + PreviewUpdated, + UnsupportedDirection { direction: ScrollDirection }, + Committed { direction: ScrollDirection, growth_rows: u32 }, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardRegistrationWithSource { +enum DownwardRegistration { NoMatch, - Matched(DownwardSampleMatch), - Ambiguous { best: DownwardSampleMatch, competing: DownwardSampleMatch }, + Matched(DirectionMatch), + Ambiguous { best: DirectionMatch, competing: DirectionMatch }, } - impl DownwardRegistration { fn map_source(self, source: DownwardSampleMatchSource) -> DownwardRegistrationWithSource { match self { @@ -5287,18 +5380,32 @@ impl DownwardRegistration { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardViewportCandidateSource { +enum DownwardSampleMatchSource { ObservedSample, PreviewOnlyLocalSample, - CommittedKeyframe, +} +impl DownwardSampleMatchSource { + const fn label(self) -> &'static str { + match self { + Self::ObservedSample => "observed_sample", + Self::PreviewOnlyLocalSample => "preview_only_local_sample", + } + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum CommittedDownwardViewportCandidateMode { - LastCommittedOnly, - IncludeRecentHistory, +enum DownwardRegistrationWithSource { + NoMatch, + Matched(DownwardSampleMatch), + Ambiguous { best: DownwardSampleMatch, competing: DownwardSampleMatch }, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DownwardViewportCandidateSource { + ObservedSample, + PreviewOnlyLocalSample, + CommittedKeyframe, +} impl DownwardViewportCandidateSource { const fn priority(self) -> u8 { match self { @@ -5339,36 +5446,223 @@ impl From for DownwardViewportCandidateSource { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DownwardViewportCandidate { - source: DownwardViewportCandidateSource, - viewport_top_y: i32, - motion_rows: u32, - mean_abs_diff_x100: u32, +enum CommittedDownwardViewportCandidateMode { + LastCommittedOnly, + IncludeRecentHistory, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct BlockedPreviewOnlyLocalCandidate { - candidate: DownwardViewportCandidate, - repeats: u8, +enum DownwardViewportResolution { + NoMatch, + Selected(DownwardViewportCandidate), + Ambiguous { preferred: DownwardViewportCandidate, competing: DownwardViewportCandidate }, } -impl DownwardViewportCandidate { - fn competing_block_reason(self, competing: Self) -> &'static str { - match (self.source, competing.source) { - ( - DownwardViewportCandidateSource::CommittedKeyframe, - DownwardViewportCandidateSource::CommittedKeyframe, - ) => "conflicting_committed_keyframe_authority", - _ => "conflicting_downward_viewport_authority", - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OverlapOrientation { + PreviousBottomToNextTop, + PreviousTopToNextBottom, +} + +#[must_use] +pub(crate) fn scroll_capture_fingerprint(image: &RgbaImage) -> Vec { + ScrollFrameFingerprint::from_image(image).into_bytes() +} + +#[must_use] +pub(crate) fn scroll_capture_fingerprint_delta(left: &[u8], right: &[u8]) -> u32 { + if left.len() != right.len() || left.is_empty() || !left.len().is_multiple_of(4) { + return u32::MAX; + } + + let mut total_abs_diff = 0_u64; + let mut comparisons = 0_u64; + + for (left_pixel, right_pixel) in left.chunks_exact(4).zip(right.chunks_exact(4)) { + total_abs_diff = total_abs_diff + .saturating_add(u64::from(left_pixel[0].abs_diff(right_pixel[0]))) + .saturating_add(u64::from(left_pixel[1].abs_diff(right_pixel[1]))) + .saturating_add(u64::from(left_pixel[2].abs_diff(right_pixel[2]))) + .saturating_add(u64::from(left_pixel[3].abs_diff(right_pixel[3]))); + comparisons = comparisons.saturating_add(4); + } + + if comparisons == 0 { u32::MAX } else { (total_abs_diff / comparisons) as u32 } +} + +#[cfg(test)] +#[must_use] +pub(crate) fn detect_vertical_overlap( + previous: &RgbaImage, + next: &RgbaImage, + config: OverlapSearchConfig, +) -> OverlapMatch { + detect_vertical_overlap_in_range( + previous, + next, + 1..=previous.height().min(next.height()), + ScrollDirection::Down, + config, + overlap_global_informative_span(previous, next), + ) +} + +pub(crate) fn compose_provisional_preview_image( + base_preview: &RgbaImage, + latest_frame: Option<&RgbaImage>, + motion_rows_hint: Option, + preview_width_px: u32, +) -> RgbaImage { + let Some(frame) = latest_frame else { + return base_preview.clone(); + }; + let Some(motion_rows_hint) = motion_rows_hint else { + return base_preview.clone(); + }; + let hinted_growth_rows = motion_rows_hint.min(frame.height()); + + if hinted_growth_rows == 0 { + return base_preview.clone(); + } + + let Some(strip) = crop_bottom_rows(frame, hinted_growth_rows) else { + return base_preview.clone(); + }; + let preview_strip = resize_strip_to_preview_width(&strip, preview_width_px); + + append_vertical_image(base_preview, &preview_strip).unwrap_or_else(|_| base_preview.clone()) +} + +fn worker_pairwise_overlap_search_config() -> OverlapSearchConfig { + OverlapSearchConfig { + min_overlap_rows: 24, + max_column_samples: 96, + max_row_samples: 96, + max_mean_abs_diff_x100: 850, + } +} + +#[cfg(target_os = "macos")] +fn classify_vision_downward_sample_motion_against( + previous: &RgbaImage, + next: &RgbaImage, +) -> Option { + let previous_cg = cg_image_from_rgba_image(previous).ok()?; + let next_cg = cg_image_from_rgba_image(next).ok()?; + let options = NSDictionary::::new(); + let request = unsafe { + VNTranslationalImageRegistrationRequest::initWithTargetedCGImage_options( + VNTranslationalImageRegistrationRequest::alloc(), + previous_cg.as_ref(), + options.as_ref(), + ) + }; + let request_array = NSArray::from_retained_slice(&[request + .clone() + .into_super() + .into_super() + .into_super() + .into_super()]); + let handler = unsafe { + VNImageRequestHandler::initWithCGImage_options( + VNImageRequestHandler::alloc(), + next_cg.as_ref(), + options.as_ref(), + ) + }; + + handler.performRequests_error(request_array.as_ref()).ok()?; + + let results = unsafe { request.results() }?; + + if results.count() == 0 { + return None; + } + + let translation = unsafe { results.objectAtIndex(0).alignmentTransform() }; + let motion_rows = translation.ty.round(); + + if !motion_rows.is_finite() || motion_rows <= 0.0 { + return None; + } + + let motion_rows = motion_rows as u32; + let config = OverlapSearchConfig::default(); + let matched = evaluate_overlap_direction( + previous, + next, + ScrollDirection::Down, + motion_rows..=motion_rows, + config, + )?; + let max_overlap = previous.height().min(next.height()); + + downward_registration_has_meaningful_overlap(matched, max_overlap, config).then_some(matched) +} + +#[cfg(not(target_os = "macos"))] +fn classify_vision_downward_sample_motion_against( + _previous: &RgbaImage, + _next: &RgbaImage, +) -> Option { + None +} + +fn estimate_pairwise_downward_shift_rows(previous: &RgbaImage, current: &RgbaImage) -> Option { + if previous.dimensions() != current.dimensions() { + return None; + } + + let (_width, height) = previous.dimensions(); + + if height < 3 { + return None; + } + + let max_shift = height.saturating_sub(1); + + evaluate_overlap_direction( + previous, + current, + ScrollDirection::Down, + 1..=max_shift, + worker_pairwise_overlap_search_config(), + ) + .map(|matched| matched.motion_rows) +} + +#[cfg(target_os = "macos")] +fn cg_image_from_rgba_image(image: &RgbaImage) -> Result> { + let width = image.width() as usize; + let height = image.height() as usize; + + if width == 0 || height == 0 { + return Err(eyre::eyre!("vision registration image has zero dimensions")); + } + + let bytes = CFData::from_bytes(image.as_raw()); + let provider = CGDataProvider::with_cf_data(Some(bytes.as_ref())) + .ok_or_else(|| eyre::eyre!("failed to create CGDataProvider for Vision registration"))?; + let color_space = CGColorSpace::new_device_rgb() + .ok_or_else(|| eyre::eyre!("failed to create RGB colorspace for Vision registration"))?; + let bitmap_info = CGBitmapInfo(CGImageAlphaInfo::Last.0 | CGImageByteOrderInfo::Order32Big.0); + + unsafe { + CGImage::new( + width, + height, + 8, + 32, + width.saturating_mul(4), + Some(color_space.as_ref()), + bitmap_info, + Some(provider.as_ref()), + ptr::null(), + false, + CGColorRenderingIntent::RenderingIntentDefault, + ) } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardViewportResolution { - NoMatch, - Selected(DownwardViewportCandidate), - Ambiguous { preferred: DownwardViewportCandidate, competing: DownwardViewportCandidate }, + .ok_or_else(|| eyre::eyre!("failed to create CGImage for Vision registration")) } fn select_downward_viewport_candidate( @@ -5447,11 +5741,9 @@ fn prefer_local_downward_viewport_candidate( .cmp(&right.mean_abs_diff_x100) .then(left.motion_rows.cmp(&right.motion_rows)) }); - let Some(committed) = committed else { return Some(local); }; - let committed_is_nearby = committed.viewport_top_y.abs_diff(local.viewport_top_y) < DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS; let committed_is_only_modestly_better = @@ -5476,167 +5768,6 @@ fn best_local_downward_viewport_candidate( }) } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct OverlapSearchRange { - start: u32, - end: u32, -} -impl OverlapSearchRange { - fn as_range(self) -> RangeInclusive { - self.start..=self.end - } -} - -impl From> for OverlapSearchRange { - fn from(range: RangeInclusive) -> Self { - Self { start: *range.start(), end: *range.end() } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectionMatchEval { - preferred_range: Option, - max_motion_rows: u32, - preferred_only_match: Option, - final_match: Option, - used_full_range_fallback: bool, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct MotionObservation { - direction: ScrollDirection, - motion_rows: u32, -} - -#[derive(Clone, Copy, Debug)] -struct UpInputMatchLog { - sample_motion: Option, - sample_down_match: Option, - sample_up_match: Option, - committed_down_match: Option, - committed_up_match: Option, - sample_override_wins: bool, - committed_override_wins: bool, -} - -#[derive(Clone, Copy, Debug)] -struct UpInputSearchWindowLog<'a> { - sample_delta: Option, - sample_down_match_eval: &'a DirectionMatchEval, - sample_up_match_eval: &'a DirectionMatchEval, - committed_down_match_eval: &'a DirectionMatchEval, - committed_up_match_eval: &'a DirectionMatchEval, - frame_equals_last_sample: bool, - frame_equals_last_committed: bool, -} - -#[derive(Clone, Copy, Debug)] -struct UpwardInputDiagnostics { - sample_down_match_eval: DirectionMatchEval, - sample_up_match_eval: DirectionMatchEval, - committed_down_match_eval: DirectionMatchEval, - committed_up_match_eval: DirectionMatchEval, - sample_override_match: Option, - committed_override_match: Option, -} - -#[derive(Clone, Copy, Debug)] -struct ResumeFrontierMatchLog { - motion_rows: u32, - candidate_observed_viewport_top_y: i32, - residual_growth_rows: u32, - raw_committed_down_match: Option, - trusted_committed_down_match: Option, - committed_up_match: Option, - frame_reacquires_last_committed_viewport: bool, -} - -#[derive(Clone, Copy, Debug)] -struct ResumeFrontierDirectMatchContext { - motion_rows: u32, - candidate_observed_viewport_top_y: i32, - residual_growth_rows: u32, -} - -#[derive(Clone, Debug)] -struct GrowthCommit { - frame: RgbaImage, - growth_rows: u32, - viewport_top_y: i32, - decision_source: &'static str, - detected_motion_rows: Option, - effective_motion_rows_hint: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct InformativeSpan { - start_x: u32, - end_exclusive_x: u32, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ScrollDirection { - Up, - Down, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ScrollObserveOutcome { - NoChange, - PreviewUpdated, - UnsupportedDirection { direction: ScrollDirection }, - Committed { direction: ScrollDirection, growth_rows: u32 }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum OverlapOrientation { - PreviousBottomToNextTop, - PreviousTopToNextBottom, -} - -#[must_use] -pub(crate) fn scroll_capture_fingerprint(image: &RgbaImage) -> Vec { - ScrollFrameFingerprint::from_image(image).into_bytes() -} - -#[must_use] -pub(crate) fn scroll_capture_fingerprint_delta(left: &[u8], right: &[u8]) -> u32 { - if left.len() != right.len() || left.is_empty() || !left.len().is_multiple_of(4) { - return u32::MAX; - } - - let mut total_abs_diff = 0_u64; - let mut comparisons = 0_u64; - - for (left_pixel, right_pixel) in left.chunks_exact(4).zip(right.chunks_exact(4)) { - total_abs_diff = total_abs_diff - .saturating_add(u64::from(left_pixel[0].abs_diff(right_pixel[0]))) - .saturating_add(u64::from(left_pixel[1].abs_diff(right_pixel[1]))) - .saturating_add(u64::from(left_pixel[2].abs_diff(right_pixel[2]))) - .saturating_add(u64::from(left_pixel[3].abs_diff(right_pixel[3]))); - comparisons = comparisons.saturating_add(4); - } - - if comparisons == 0 { u32::MAX } else { (total_abs_diff / comparisons) as u32 } -} - -#[cfg(test)] -#[must_use] -pub(crate) fn detect_vertical_overlap( - previous: &RgbaImage, - next: &RgbaImage, - config: OverlapSearchConfig, -) -> OverlapMatch { - detect_vertical_overlap_in_range( - previous, - next, - 1..=previous.height().min(next.height()), - ScrollDirection::Down, - config, - overlap_global_informative_span(previous, next), - ) -} - fn evaluate_overlap_direction( previous: &RgbaImage, next: &RgbaImage, @@ -5657,7 +5788,6 @@ fn collect_overlap_direction_matches( let Some(informative_span) = overlap_global_informative_span(previous, next) else { return Vec::new(); }; - let max_overlap = previous.height().min(next.height()); let effective_min_overlap = if max_overlap <= config.min_overlap_rows { 1 } else { config.min_overlap_rows.max(1) }; @@ -5698,6 +5828,7 @@ fn collect_overlap_direction_matches( .cmp(&right.mean_abs_diff_x100) .then(left.motion_rows.cmp(&right.motion_rows)) }); + matches } @@ -5739,6 +5870,7 @@ fn collect_overlap_direction_matches_in_ranges( if matched.mean_abs_diff_x100 < previous.mean_abs_diff_x100 { *previous = matched; } + continue; } @@ -5980,31 +6112,6 @@ fn resize_strip_to_preview_width(strip: &RgbaImage, preview_width_px: u32) -> Rg imageops::resize(strip, preview_width_px, preview_height, FilterType::Triangle) } -pub(crate) fn compose_provisional_preview_image( - base_preview: &RgbaImage, - latest_frame: Option<&RgbaImage>, - motion_rows_hint: Option, - preview_width_px: u32, -) -> RgbaImage { - let Some(frame) = latest_frame else { - return base_preview.clone(); - }; - let Some(motion_rows_hint) = motion_rows_hint else { - return base_preview.clone(); - }; - let hinted_growth_rows = motion_rows_hint.min(frame.height()); - if hinted_growth_rows == 0 { - return base_preview.clone(); - } - - let Some(strip) = crop_bottom_rows(frame, hinted_growth_rows) else { - return base_preview.clone(); - }; - let preview_strip = resize_strip_to_preview_width(&strip, preview_width_px); - - append_vertical_image(base_preview, &preview_strip).unwrap_or_else(|_| base_preview.clone()) -} - fn crop_bottom_rows(frame: &RgbaImage, rows: u32) -> Option { let rows = rows.min(frame.height()); @@ -6220,8 +6327,6 @@ mod tests { DownwardViewportCandidate, DownwardViewportCandidateSource, DownwardViewportResolution, GrowthCommit, MotionObservation, OverlapSearchConfig, PreviewOnlyDownwardLocalSample, ScrollDirection, ScrollFrameFingerprint, ScrollObserveOutcome, ScrollSession, - classify_vision_downward_sample_motion_against, estimate_pairwise_downward_shift_rows, - select_downward_viewport_candidate, }; fn make_test_image(width: u32, rows: &[[u8; 4]]) -> image::RgbaImage { @@ -6272,18 +6377,17 @@ mod tests { start_row: u32, thumb_top: u32, ) -> image::RgbaImage { - let mut image = make_sparse_textlike_window(width, height, start_row); let track_left = width.saturating_sub(18); let thumb_height = (height / 4).max(12).min(height.max(1)); let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); let thumb_right = width.saturating_sub(3).max(track_left.saturating_add(4)); + let mut image = make_sparse_textlike_window(width, height, start_row); for y in 0..height { for x in track_left..width { image.put_pixel(x, y, Rgba([224, 224, 224, 255])); } } - for y in thumb_top..thumb_top.saturating_add(thumb_height) { for x in track_left.saturating_add(3)..thumb_right { image.put_pixel(x, y, Rgba([28, 28, 28, 255])); @@ -6294,12 +6398,12 @@ mod tests { } fn make_browser_like_window(width: u32, height: u32, start_row: u32) -> image::RgbaImage { - let mut image = make_sparse_textlike_window(width, height, start_row); let scrollbar_left = width.saturating_sub(18); let content_left = 56_u32; let content_right = width.saturating_sub(48); let heading_width = 220_u32; let paragraph_width = content_right.saturating_sub(content_left); + let mut image = make_sparse_textlike_window(width, height, start_row); for y in 0..height { let document_row = start_row.saturating_add(y); @@ -6311,19 +6415,19 @@ mod tests { } else if document_row % 420 >= 54 && document_row % 420 < 220 { if document_row % 24 < 3 { let trim = ((document_row / 24) % 5) * 18; + for x in content_left ..content_left.saturating_add(paragraph_width.saturating_sub(trim)) { image.put_pixel(x, y, Rgba([72, 72, 72, 255])); } } - } else if document_row % 420 >= 270 && document_row % 420 < 360 { - if document_row % 20 < 2 { - for x in content_left.saturating_add(20) - ..content_left.saturating_add(paragraph_width.saturating_sub(70)) - { - image.put_pixel(x, y, Rgba([98, 98, 98, 255])); - } + } else if document_row % 420 >= 270 && document_row % 420 < 360 && document_row % 20 < 2 + { + for x in content_left.saturating_add(20) + ..content_left.saturating_add(paragraph_width.saturating_sub(70)) + { + image.put_pixel(x, y, Rgba([98, 98, 98, 255])); } } @@ -6335,6 +6439,7 @@ mod tests { let thumb_height = (height / 5).max(16); let thumb_top = (start_row / 3) % height.max(thumb_height + 1); let thumb_top = thumb_top.min(height.saturating_sub(thumb_height)); + for y in thumb_top..thumb_top.saturating_add(thumb_height) { for x in scrollbar_left.saturating_add(3)..width.saturating_sub(4) { image.put_pixel(x, y, Rgba([96, 96, 96, 255])); @@ -6416,7 +6521,7 @@ mod tests { fn worker_pairwise_vision_commits_substantial_downward_growth_with_corroboration() { let base = make_sparse_textlike_window(512, 640, 0); let moved = make_sparse_textlike_window(512, 640, 90); - let matched = classify_vision_downward_sample_motion_against(&base, &moved) + let matched = scroll_capture::classify_vision_downward_sample_motion_against(&base, &moved) .expect("vision registration should detect the substantial downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); let outcome = session.observe_worker_pairwise_vision_frame(moved).unwrap(); @@ -6438,7 +6543,7 @@ mod tests { let base = make_sparse_textlike_window(512, 640, 0); let moved = make_sparse_textlike_window(512, 640, 58); - assert_eq!(estimate_pairwise_downward_shift_rows(&base, &moved), Some(58)); + assert_eq!(scroll_capture::estimate_pairwise_downward_shift_rows(&base, &moved), Some(58)); } #[test] @@ -6446,7 +6551,7 @@ mod tests { let base = make_browser_like_window(512, 640, 0); let moved = make_browser_like_window(512, 640, 320); - assert_eq!(estimate_pairwise_downward_shift_rows(&base, &moved), Some(320)); + assert_eq!(scroll_capture::estimate_pairwise_downward_shift_rows(&base, &moved), Some(320)); } #[test] @@ -6457,7 +6562,10 @@ mod tests { .collect::>(); for window in frames.windows(2) { - assert_eq!(estimate_pairwise_downward_shift_rows(&window[0], &window[1]), Some(180)); + assert_eq!( + scroll_capture::estimate_pairwise_downward_shift_rows(&window[0], &window[1]), + Some(180) + ); } } @@ -6467,10 +6575,12 @@ mod tests { let base = make_sparse_textlike_window(512, 640, 0); let step_one = make_sparse_textlike_window(512, 640, 180); let step_two = make_sparse_textlike_window(512, 640, 360); - let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) - .expect("first pairwise registration should detect downward motion"); - let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) - .expect("followup pairwise registration should detect downward motion"); + let first_match = + scroll_capture::classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first pairwise registration should detect downward motion"); + let followup_match = + scroll_capture::classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup pairwise registration should detect downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); assert_eq!( @@ -6503,10 +6613,12 @@ mod tests { let base = make_sparse_textlike_window(512, 640, 0); let step_one = make_sparse_textlike_window(512, 640, 180); let step_two = make_sparse_textlike_window(512, 640, 360); - let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) - .expect("first pairwise registration should detect downward motion"); - let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) - .expect("followup pairwise registration should detect downward motion"); + let first_match = + scroll_capture::classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first pairwise registration should detect downward motion"); + let followup_match = + scroll_capture::classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup pairwise registration should detect downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); assert_eq!( @@ -6539,9 +6651,11 @@ mod tests { let base = make_browser_like_window(512, 640, 0); let blocked = make_browser_like_window(512, 640, 760); let followup = make_browser_like_window(512, 640, 844); - let matched = classify_vision_downward_sample_motion_against(&blocked, &followup).expect( - "pairwise registration should detect the followup step after the blocked overshot", - ); + let matched = + scroll_capture::classify_vision_downward_sample_motion_against(&blocked, &followup) + .expect( + "pairwise registration should detect the followup step after the blocked overshot", + ); let mut session = ScrollSession::new(base, 320).unwrap(); assert_eq!( @@ -6566,7 +6680,9 @@ mod tests { fn worker_pairwise_vision_clears_preview_local_followup_carryover_on_no_change() { let base = make_sparse_textlike_window(512, 640, 0); let mut session = ScrollSession::new(base.clone(), 320).unwrap(); + session.record_preview_only_downward_local_sample(&base, 123); + session.pending_suppressed_huge_preview_only_local_followup = Some(DownwardViewportCandidate { source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, @@ -6600,10 +6716,12 @@ mod tests { fn worker_pairwise_vision_clears_preview_local_followup_carryover_on_commit() { let base = make_sparse_textlike_window(512, 640, 0); let moved = make_sparse_textlike_window(512, 640, 180); - let matched = classify_vision_downward_sample_motion_against(&base, &moved) + let matched = scroll_capture::classify_vision_downward_sample_motion_against(&base, &moved) .expect("pairwise registration should detect downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); + session.record_preview_only_downward_local_sample(&moved, 180); + session.pending_suppressed_huge_preview_only_local_followup = Some(DownwardViewportCandidate { source: DownwardViewportCandidateSource::PreviewOnlyLocalSample, @@ -6649,8 +6767,9 @@ mod tests { for window in frames.windows(2) { let previous = &window[0]; let next = window[1].clone(); - let matched = classify_vision_downward_sample_motion_against(previous, &next) - .expect("pairwise registration should detect each slowdown step"); + let matched = + scroll_capture::classify_vision_downward_sample_motion_against(previous, &next) + .expect("pairwise registration should detect each slowdown step"); assert_eq!( session.observe_worker_pairwise_vision_frame(next).unwrap(), @@ -6673,7 +6792,7 @@ mod tests { fn worker_pairwise_vision_commits_browser_like_growth_above_legacy_cap() { let base = make_browser_like_window(512, 640, 0); let moved = make_browser_like_window(512, 640, 320); - let matched = classify_vision_downward_sample_motion_against(&base, &moved) + let matched = scroll_capture::classify_vision_downward_sample_motion_against(&base, &moved) .expect("vision registration should detect the browser-like downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); @@ -6703,8 +6822,9 @@ mod tests { for window in frames.windows(2) { let previous = &window[0]; let next = window[1].clone(); - let matched = classify_vision_downward_sample_motion_against(previous, &next) - .expect("pairwise registration should detect each browser-like step"); + let matched = + scroll_capture::classify_vision_downward_sample_motion_against(previous, &next) + .expect("pairwise registration should detect each browser-like step"); assert_eq!( session.observe_worker_pairwise_vision_frame(next).unwrap(), @@ -6728,10 +6848,12 @@ mod tests { let base = make_browser_like_window(512, 640, 0); let step_one = make_browser_like_window(512, 640, 180); let step_two = make_browser_like_window(512, 640, 360); - let first_match = classify_vision_downward_sample_motion_against(&base, &step_one) - .expect("first browser-like step should register downward motion"); - let followup_match = classify_vision_downward_sample_motion_against(&step_one, &step_two) - .expect("followup browser-like step should register downward motion"); + let first_match = + scroll_capture::classify_vision_downward_sample_motion_against(&base, &step_one) + .expect("first browser-like step should register downward motion"); + let followup_match = + scroll_capture::classify_vision_downward_sample_motion_against(&step_one, &step_two) + .expect("followup browser-like step should register downward motion"); let mut session = ScrollSession::new(base, 320).unwrap(); assert_eq!( @@ -6764,9 +6886,11 @@ mod tests { let base = make_browser_like_window(512, 640, 0); let blocked = make_browser_like_window(512, 640, 700); let followup = make_browser_like_window(512, 640, 784); - let matched = classify_vision_downward_sample_motion_against(&blocked, &followup).expect( - "browser-like pairwise registration should use the immediately previous worker frame", - ); + let matched = + scroll_capture::classify_vision_downward_sample_motion_against(&blocked, &followup) + .expect( + "browser-like pairwise registration should use the immediately previous worker frame", + ); let mut session = ScrollSession::new(base, 320).unwrap(); assert_eq!( @@ -7151,8 +7275,10 @@ mod tests { | ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange )); + let resume_outcome = session.observe_downward_sample(make_window(&document, 3, 1, 5)).unwrap(); + assert!(matches!( resume_outcome, ScrollObserveOutcome::NoChange @@ -7247,8 +7373,10 @@ mod tests { | ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } )); assert_eq!(session.export_image().height(), height_after_upward_rewind); + let partial_resume_outcome = session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(); + assert!(matches!( partial_resume_outcome, ScrollObserveOutcome::NoChange @@ -7293,8 +7421,10 @@ mod tests { | ScrollObserveOutcome::PreviewUpdated | ScrollObserveOutcome::NoChange )); + let return_outcome = session.observe_downward_sample(make_window(&document, 3, 2, 5)).unwrap(); + assert!(matches!( return_outcome, ScrollObserveOutcome::NoChange @@ -7370,7 +7500,7 @@ mod tests { let mut candidates = [observed, committed]; assert_eq!( - select_downward_viewport_candidate(&mut candidates), + scroll_capture::select_downward_viewport_candidate(&mut candidates), DownwardViewportResolution::Ambiguous { preferred: committed, competing: observed } ); } @@ -7412,6 +7542,7 @@ mod tests { session.last_committed_frame = image::RgbaImage::from_pixel(256, 120, Rgba([255, 255, 255, 255])); + let target = make_sparse_textlike_window(256, 120, 39); let mut candidates = Vec::new(); @@ -7439,6 +7570,7 @@ mod tests { session.last_committed_frame = image::RgbaImage::from_pixel(256, 120, Rgba([255, 255, 255, 255])); + let target = make_sparse_textlike_window(256, 120, 39); let mut candidates = Vec::new(); @@ -7488,7 +7620,7 @@ mod tests { let mut candidates = [observed, committed]; assert_eq!( - select_downward_viewport_candidate(&mut candidates), + scroll_capture::select_downward_viewport_candidate(&mut candidates), DownwardViewportResolution::Selected(observed) ); } @@ -7646,6 +7778,7 @@ mod tests { session.last_motion_rows_hint = Some(1); session.transient_motion_rows_hint = Some(277); session.transient_burst_search_enabled = true; + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 74), growth_rows: 1, @@ -7840,6 +7973,7 @@ mod tests { session.last_observed_sample_registration_motion_rows = Some(20); session.last_downward_viewport_candidates_before_prune = Some("PreviewOnlyLocalSample@472/20:0,CommittedKeyframe@472/20:0".to_string()); + for (viewport_top_y, growth_rows) in [(442_i32, 8_u32), (452_i32, 10_u32)] { session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window( @@ -7880,6 +8014,7 @@ mod tests { session.last_observed_sample_registration_motion_rows = Some(24); session.last_downward_viewport_candidates_before_prune = Some("PreviewOnlyLocalSample@261/24:329,CommittedKeyframe@512/275:460".to_string()); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 237), growth_rows: 20, @@ -8061,6 +8196,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { frame: previous, growth_rows: 4, @@ -8096,6 +8232,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 416 }); + session.growth_history.push(GrowthCommit { frame: previous, growth_rows: 10, @@ -8131,6 +8268,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { frame: previous, growth_rows: 12, @@ -8266,6 +8404,7 @@ mod tests { session.last_observed_sample_registration_motion_rows = Some(164); session.last_preview_only_local_registration_result = Some("matched"); session.last_preview_only_local_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 230), growth_rows: 18, @@ -8303,6 +8442,7 @@ mod tests { session.last_observed_sample_registration_motion_rows = Some(112); session.last_preview_only_local_registration_result = Some("matched"); session.last_preview_only_local_registration_motion_rows = Some(276); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 230), growth_rows: 18, @@ -8340,6 +8480,7 @@ mod tests { session.last_observed_sample_registration_motion_rows = Some(38); session.last_preview_only_local_registration_result = Some("matched"); session.last_preview_only_local_registration_motion_rows = Some(38); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 230), growth_rows: 18, @@ -8375,6 +8516,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_observed_sample_registration_result = Some("matched"); session.last_observed_sample_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 230), growth_rows: 18, @@ -8415,6 +8557,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_observed_sample_registration_result = Some("matched"); session.last_observed_sample_registration_motion_rows = Some(164); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 230), growth_rows: 18, @@ -8566,6 +8709,7 @@ mod tests { fn suppressed_huge_preview_local_followup_block_budget_scales_with_far_recovery_ratio() { let mut session = ScrollSession::new(make_sparse_textlike_window(256, 120, 0), 320).unwrap(); + session.last_motion_rows_hint = Some(18); assert_eq!( @@ -8606,14 +8750,17 @@ mod tests { motion_rows: 164, mean_abs_diff_x100: 0, }); + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); session.pending_suppressed_huge_preview_only_local_followup = None; session.blocked_followup_after_suppressed_huge_preview_local_jump = true; + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); session.blocked_followup_after_suppressed_huge_preview_local_jump = false; session.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; + assert!(session.should_refresh_downward_observed_baseline_after_huge_suppressed_jump()); } @@ -8626,10 +8773,12 @@ mod tests { &make_sparse_textlike_window(256, 120, 32), Some(32), ); + assert!(session.last_preview_only_downward_local_sample.is_some()); assert!(!session.should_reset_preview_only_local_baseline_after_huge_far_committed_block()); session.blocked_far_committed_only_recovery_after_corroborated_huge_local_jump = true; + assert!(session.should_reset_preview_only_local_baseline_after_huge_far_committed_block()); } @@ -8721,6 +8870,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { frame: previous.clone(), growth_rows: 4, @@ -8755,6 +8905,7 @@ mod tests { session.transient_burst_search_enabled = true; session.last_preview_only_downward_local_sample = Some(PreviewOnlyDownwardLocalSample { frame: previous.clone(), viewport_top_y: 145 }); + session.growth_history.push(GrowthCommit { frame: previous.clone(), growth_rows: 12, @@ -8989,7 +9140,7 @@ mod tests { session.last_preview_only_local_registration_result = Some("matched"); session.last_preview_only_local_registration_motion_rows = Some(272); - let candidates = vec![DownwardViewportCandidate { + let candidates = [DownwardViewportCandidate { source: DownwardViewportCandidateSource::CommittedKeyframe, viewport_top_y: 220, motion_rows: 32, @@ -9081,6 +9232,7 @@ mod tests { session.last_observed_sample_registration_result = Some("matched"); session.last_observed_sample_registration_motion_rows = Some(135); session.last_preview_only_local_registration_result = Some("no_match"); + session.growth_history.push(super::GrowthCommit { frame: make_sparse_textlike_window(256, 120, 237), growth_rows: 20, @@ -9121,7 +9273,7 @@ mod tests { session.last_preview_only_local_registration_result = Some("matched"); session.last_preview_only_local_registration_motion_rows = Some(272); - let candidates = vec![DownwardViewportCandidate { + let candidates = [DownwardViewportCandidate { source: DownwardViewportCandidateSource::CommittedKeyframe, viewport_top_y: 500, motion_rows: 310, From 9c698846a3d31cb92e2dd312ee7daef005c8da9f Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:08:31 +0800 Subject: [PATCH 05/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-ci","summary":"repair linux cfg surface for review batch","intent":"keep PR #47 green on Linux CI after scroll-capture review fixes","impact":"restore cross-platform compile surfaces for overlay scroll-capture repair code","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 41 +++++++++++-------- .../src/overlay/scroll_runtime.rs | 16 ++++++-- .../src/overlay/session_state.rs | 4 +- packages/rsnap-overlay/src/scroll_capture.rs | 2 + 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index e712051f..019d0c70 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -8,11 +8,13 @@ mod session_state; mod trace_recording; mod window_runtime; +#[cfg(target_os = "macos")] use std::collections::VecDeque; #[cfg(target_os = "macos")] use std::ffi::c_void; use std::mem; use std::panic; +#[cfg(target_os = "macos")] use std::ptr; use std::slice; #[cfg(target_os = "macos")] @@ -494,19 +496,13 @@ impl FrozenToolbarTool { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ScrollCaptureFrameSource { - Worker { - request_id: u64, - }, - #[cfg(target_os = "macos")] - LiveStream { - frame_seq: u64, - }, + Worker { request_id: u64 }, + LiveStream { frame_seq: u64 }, } impl ScrollCaptureFrameSource { const fn as_str(self) -> &'static str { match self { Self::Worker { .. } => "worker", - #[cfg(target_os = "macos")] Self::LiveStream { .. } => "live_stream", } } @@ -514,7 +510,6 @@ impl ScrollCaptureFrameSource { const fn worker_request_id(self) -> Option { match self { Self::Worker { request_id } => Some(request_id), - #[cfg(target_os = "macos")] Self::LiveStream { .. } => None, } } @@ -2050,7 +2045,15 @@ impl OverlaySession { } if let Some(monitor) = self.scroll_capture.monitor.or(self.state.monitor) { - self.position_scroll_preview_window(monitor); + #[cfg(target_os = "macos")] + { + self.position_scroll_preview_window(monitor); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = monitor; + } } } @@ -5456,7 +5459,6 @@ impl OverlaySession { now.saturating_duration_since(input_direction_at) <= SCROLL_CAPTURE_INPUT_FRESHNESS } - #[cfg(target_os = "macos")] fn scroll_capture_has_fresh_downward_backlog_at(&self, now: Instant) -> bool { if self.scroll_capture.input_direction != Some(ScrollDirection::Down) || self.scroll_capture.downward_motion_rows_pending <= 0.0 @@ -5483,7 +5485,6 @@ impl OverlaySession { }) } - #[cfg(target_os = "macos")] fn scroll_capture_should_allow_post_stall_burst_search_at( &self, frame_seq: u64, @@ -5843,7 +5844,6 @@ impl OverlaySession { .scroll_capture .external_scroll_input_drain_reader .clone(), - #[cfg(target_os = "macos")] last_external_scroll_input_seq: 0, #[cfg(target_os = "macos")] pixel_delta_residual: MacOSScrollPixelResidual::default(), @@ -5856,7 +5856,6 @@ impl OverlaySession { )), #[cfg(target_os = "macos")] live_stream_backlog: VecDeque::new(), - #[cfg(target_os = "macos")] last_stream_frame_seq: 0, #[cfg(target_os = "macos")] last_stream_frame_fingerprint: None, @@ -5870,7 +5869,6 @@ impl OverlaySession { last_stream_poll_at: None, #[cfg(target_os = "macos")] last_duplicate_stream_refresh_at: None, - #[cfg(target_os = "macos")] pending_post_stall_burst_after_seq: None, #[cfg(target_os = "macos")] live_stream_stale_grace: None, @@ -7260,6 +7258,14 @@ impl OverlaySession { OverlayExit::Saved(path) => ("saved", None, Some(path.display().to_string()), None), OverlayExit::Error(message) => ("error", None, None, Some(message.as_str())), }; + #[cfg(target_os = "macos")] + let scroll_capture_has_live_stream = self.scroll_capture.live_stream.is_some(); + #[cfg(not(target_os = "macos"))] + let scroll_capture_has_live_stream = false; + #[cfg(target_os = "macos")] + let live_sample_stream_present = self.live_sample_stream.is_some(); + #[cfg(not(target_os = "macos"))] + let live_sample_stream_present = false; tracing::info!( op = "overlay.exit_begin", @@ -7268,8 +7274,8 @@ impl OverlaySession { saved_path, error_message, scroll_capture_active = self.scroll_capture.active, - scroll_capture_has_live_stream = self.scroll_capture.live_stream.is_some(), - live_sample_stream_present = self.live_sample_stream.is_some(), + scroll_capture_has_live_stream, + live_sample_stream_present, last_event_phase = %self.event_loop_phase.as_str(), last_event_window_id = ?self.event_loop_last_progress_window_id, last_event_monitor_id = ?self.event_loop_last_progress_monitor_id, @@ -12973,6 +12979,7 @@ mod tests { use std::collections::VecDeque; #[cfg(target_os = "macos")] use std::sync::Arc; + #[cfg(target_os = "macos")] use std::thread; #[cfg(target_os = "macos")] use std::time::Duration; diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index a9c2f86f..3aa463f0 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -5,9 +5,12 @@ use image::RgbaImage; #[cfg(target_os = "macos")] use crate::live_frame_stream_macos::MacLiveFrameStream; +#[cfg(target_os = "macos")] use crate::overlay::SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL; use crate::overlay::SCROLL_CAPTURE_SAMPLE_INTERVAL; #[cfg(target_os = "macos")] +use crate::overlay::ScrollCaptureTraceInputRecord; +#[cfg(target_os = "macos")] use crate::overlay::session_state::ScrollCaptureLiveFrame; #[cfg(target_os = "macos")] use crate::overlay::{ @@ -19,7 +22,7 @@ use crate::overlay::{ use crate::overlay::{MonitorRect, RectPoints}; use crate::overlay::{ OverlayControl, OverlaySession, ScrollCaptureFrameSource, ScrollCaptureTraceFrameRecord, - ScrollCaptureTraceInputRecord, ScrollObserveOutcome, ScrollSession, + ScrollObserveOutcome, ScrollSession, }; use crate::scroll_capture::ScrollDirection; #[cfg(target_os = "macos")] @@ -630,7 +633,6 @@ impl OverlaySession { self.consume_scroll_capture_backlog(max_frames); } - #[cfg(target_os = "macos")] pub(super) fn replay_recorded_live_stream_frame( &mut self, frame: RgbaImage, @@ -638,13 +640,16 @@ impl OverlaySession { observed_at: Instant, allow_stale_input: bool, ) -> Option> { + #[cfg(target_os = "macos")] if self.scroll_capture_should_arm_post_stall_burst_for_time_gap_at(observed_at) { self.scroll_capture.pending_post_stall_burst_after_seq = Some(frame_seq.saturating_sub(1)); } + #[cfg(target_os = "macos")] let frame_for_activity = ScrollCaptureLiveFrame { frame_seq, captured_at: observed_at, image: frame.clone() }; + #[cfg(target_os = "macos")] let _ = self.note_scroll_capture_live_stream_frame_activity(&frame_for_activity); self.scroll_capture.last_stream_frame_seq = frame_seq; @@ -656,9 +661,12 @@ impl OverlaySession { observed_at, ); - self.scroll_capture.last_consumed_stream_frame_captured_at = Some(observed_at); + #[cfg(target_os = "macos")] + { + self.scroll_capture.last_consumed_stream_frame_captured_at = Some(observed_at); - self.maybe_schedule_duplicate_stream_refresh(frame_seq, observed_at); + self.maybe_schedule_duplicate_stream_refresh(frame_seq, observed_at); + } outcome } diff --git a/packages/rsnap-overlay/src/overlay/session_state.rs b/packages/rsnap-overlay/src/overlay/session_state.rs index 02b2fdfa..540fc31d 100644 --- a/packages/rsnap-overlay/src/overlay/session_state.rs +++ b/packages/rsnap-overlay/src/overlay/session_state.rs @@ -1,3 +1,4 @@ +#[cfg(target_os = "macos")] use std::collections::VecDeque; use std::{ collections::HashMap, @@ -191,7 +192,6 @@ pub(super) struct ScrollCaptureState { pub(super) overlay_mouse_passthrough_until: Option, #[cfg(target_os = "macos")] pub(super) external_scroll_input_drain_reader: Option, - #[cfg(target_os = "macos")] pub(super) last_external_scroll_input_seq: u64, #[cfg(target_os = "macos")] pub(super) pixel_delta_residual: MacOSScrollPixelResidual, @@ -199,7 +199,6 @@ pub(super) struct ScrollCaptureState { pub(super) live_stream: Option, #[cfg(target_os = "macos")] pub(super) live_stream_backlog: VecDeque, - #[cfg(target_os = "macos")] pub(super) last_stream_frame_seq: u64, #[cfg(target_os = "macos")] pub(super) last_stream_frame_fingerprint: Option>, @@ -213,7 +212,6 @@ pub(super) struct ScrollCaptureState { pub(super) last_stream_poll_at: Option, #[cfg(target_os = "macos")] pub(super) last_duplicate_stream_refresh_at: Option, - #[cfg(target_os = "macos")] pub(super) pending_post_stall_burst_after_seq: Option, #[cfg(target_os = "macos")] pub(super) live_stream_stale_grace: Option, diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 61ee6a1c..e4beec9b 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -257,6 +257,7 @@ pub mod bench_support { } use std::ops::RangeInclusive; +#[cfg(target_os = "macos")] use std::ptr; use color_eyre::eyre::{self, Result}; @@ -268,6 +269,7 @@ use image::{ use objc2::{AnyThread, runtime::AnyObject}; #[cfg(target_os = "macos")] use objc2_core_foundation::CFData; +#[cfg(target_os = "macos")] use objc2_core_foundation::CFRetained; #[cfg(target_os = "macos")] use objc2_core_graphics::{ From 8906564f8bebab21171b075ac3f1d9a59bb2bed6 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:16:17 +0800 Subject: [PATCH 06/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-ci","summary":"repair remaining linux lint leaks","intent":"clear the last Rust checks failure on PR #47","impact":"align cross-platform imports and trace enum coverage with Linux clippy","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/backend.rs | 4 +++- packages/rsnap-overlay/src/overlay.rs | 6 +++--- packages/rsnap-overlay/src/overlay/scroll_runtime.rs | 4 +++- packages/rsnap-overlay/src/overlay/trace_recording.rs | 1 - 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index 04b27bb8..99a88264 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -1437,7 +1437,9 @@ fn xcap_find_monitor(monitor: MonitorRect) -> Result { #[cfg(test)] mod tests { - use crate::backend::{self, CaptureBackend, StubCaptureBackend}; + #[cfg(target_os = "macos")] + use crate::backend::{self}; + use crate::backend::{CaptureBackend, StubCaptureBackend}; #[cfg(target_os = "macos")] use crate::state::{GlobalPoint, MonitorRect, RectPoints}; diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 019d0c70..02258e6f 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -14,7 +14,6 @@ use std::collections::VecDeque; use std::ffi::c_void; use std::mem; use std::panic; -#[cfg(target_os = "macos")] use std::ptr; use std::slice; #[cfg(target_os = "macos")] @@ -133,9 +132,10 @@ use self::session_state::{ LiveStreamStaleGrace, MacOSHudWindowConfigState, MacOSScrollPixelResidual, MacOSScrollWheelEvent, }; +#[cfg(target_os = "macos")] +use self::trace_recording::ScrollCaptureTraceInputRecord; use self::trace_recording::{ - ScrollCaptureTraceFrameRecord, ScrollCaptureTraceInputRecord, ScrollCaptureTraceRecorder, - ScrollCaptureTraceSessionSnapshot, + ScrollCaptureTraceFrameRecord, ScrollCaptureTraceRecorder, ScrollCaptureTraceSessionSnapshot, }; #[cfg(target_os = "macos")] use crate::backend; diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index 3aa463f0..f7f02aa4 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -1,4 +1,6 @@ -use std::time::{Duration, Instant}; +#[cfg(target_os = "macos")] +use std::time::Duration; +use std::time::Instant; use color_eyre::Result; use image::RgbaImage; diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index 8e775381..ffb5c50e 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -116,7 +116,6 @@ impl From for ScrollCaptureTraceFrameSource { fn from(value: ScrollCaptureFrameSource) -> Self { match value { ScrollCaptureFrameSource::Worker { request_id } => Self::Worker { request_id }, - #[cfg(target_os = "macos")] ScrollCaptureFrameSource::LiveStream { frame_seq } => Self::LiveStream { frame_seq }, } } From 3e99bdd426140ed1e18132ef941fa33b24b0a6b1 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:23:10 +0800 Subject: [PATCH 07/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-ci","summary":"isolate macos-only tracing boundary","intent":"clear Linux Rust checks by shrinking macOS-only scroll-capture surfaces","impact":"restore Ubuntu lint by cfg-gating tracing and retry symbols to their real platform boundary","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 2 ++ .../src/overlay/scroll_runtime.rs | 21 ++++++++++------ .../src/overlay/session_state.rs | 1 + .../src/overlay/trace_recording.rs | 24 ++++++++++++++++--- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 02258e6f..fe2e7767 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -307,6 +307,7 @@ const SCROLL_PREVIEW_WINDOW_WIDTH_POINTS: f64 = 260.0; const SCROLL_PREVIEW_WINDOW_HEIGHT_POINTS: f64 = 360.0; const SCROLL_PREVIEW_WINDOW_MARGIN_POINTS: i32 = 16; const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); +#[cfg(target_os = "macos")] const SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL: Duration = Duration::from_millis(60); #[cfg(target_os = "macos")] const SCROLL_CAPTURE_MOUSE_PASSTHROUGH_IDLE_GRACE: Duration = Duration::from_millis(180); @@ -5827,6 +5828,7 @@ impl OverlaySession { active: true, paused: false, monitor: Some(monitor), + #[cfg(target_os = "macos")] capture_rect_points: Some(capture_rect_points), capture_rect_pixels: Some(capture_rect_pixels), input_direction: None, diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index f7f02aa4..bdee0b05 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -864,13 +864,20 @@ impl OverlaySession { let preview_frame = frame.clone(); let frame_px = frame.dimensions(); let prior_block_reason = self.scroll_capture_observation_block_reason_at(observation_at); - #[cfg(target_os = "macos")] - let allow_stale_input = allow_stale_input - || prior_block_reason == Some("stale_input") - && matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) - && self.consume_live_stream_stale_grace_if_current(); - #[cfg(not(target_os = "macos"))] - let allow_stale_input = allow_stale_input; + let allow_stale_input = { + #[cfg(target_os = "macos")] + { + allow_stale_input + || prior_block_reason == Some("stale_input") + && matches!(source, ScrollCaptureFrameSource::LiveStream { .. }) + && self.consume_live_stream_stale_grace_if_current() + } + + #[cfg(not(target_os = "macos"))] + { + allow_stale_input + } + }; if let Some(reason) = prior_block_reason { let input_age_ms = self.scroll_capture_input_age_ms_at(observation_at); diff --git a/packages/rsnap-overlay/src/overlay/session_state.rs b/packages/rsnap-overlay/src/overlay/session_state.rs index 540fc31d..44f445de 100644 --- a/packages/rsnap-overlay/src/overlay/session_state.rs +++ b/packages/rsnap-overlay/src/overlay/session_state.rs @@ -178,6 +178,7 @@ pub(super) struct ScrollCaptureState { pub(super) active: bool, pub(super) paused: bool, pub(super) monitor: Option, + #[cfg(target_os = "macos")] pub(super) capture_rect_points: Option, pub(super) capture_rect_pixels: Option, pub(super) input_direction: Option, diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index ffb5c50e..5947650c 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -1,11 +1,18 @@ +#[cfg(any(target_os = "macos", test))] use std::{ - env, fs, - path::{Path, PathBuf}, + env, + path::Path, process, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{SystemTime, UNIX_EPOCH}, +}; +use std::{ + fs, + path::PathBuf, + time::{Duration, Instant}, }; use color_eyre::eyre::{self, Result, WrapErr}; +#[cfg(any(target_os = "macos", test))] use directories::ProjectDirs; use image::RgbaImage; use serde::{Deserialize, Serialize}; @@ -17,8 +24,11 @@ use crate::{ scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}, }; +#[cfg(any(target_os = "macos", test))] const SCROLL_CAPTURE_TRACE_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE"; +#[cfg(any(target_os = "macos", test))] const SCROLL_CAPTURE_TRACE_DIR_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE_DIR"; +#[cfg(any(target_os = "macos", test))] const SCROLL_CAPTURE_TRACE_SCHEMA: &str = "scroll_capture_live_trace/1"; const SCROLL_CAPTURE_TRACE_MANIFEST_FLUSH_INTERVAL: Duration = Duration::from_millis(250); @@ -235,6 +245,7 @@ pub(crate) struct ScrollCaptureTraceRecorder { manifest: ScrollCaptureLiveTraceManifest, } impl ScrollCaptureTraceRecorder { + #[cfg(any(target_os = "macos", test))] pub(crate) fn from_env( monitor: MonitorRect, capture_rect_pixels: RectPoints, @@ -263,6 +274,7 @@ impl ScrollCaptureTraceRecorder { } } + #[cfg(any(target_os = "macos", test))] pub(crate) fn record_replayed_input(&mut self, record: ScrollCaptureTraceInputRecord) { self.manifest.entries.push(ScrollCaptureLiveTraceEntry::Input( ScrollCaptureTraceInputEntry { @@ -376,10 +388,12 @@ impl ScrollCaptureTraceRecorder { self.flush_manifest_best_effort("finalize_session"); } + #[cfg(any(target_os = "macos", test))] pub(crate) fn manifest_path(&self) -> &Path { &self.manifest_path } + #[cfg(any(target_os = "macos", test))] pub(crate) fn new_for_root_dir( trace_root: PathBuf, monitor: MonitorRect, @@ -500,6 +514,7 @@ impl Drop for ScrollCaptureTraceRecorder { } } +#[cfg(any(target_os = "macos", test))] pub(crate) struct ScrollCaptureTraceInputRecord { pub(crate) seq: u64, pub(crate) cursor_global: (f64, f64), @@ -560,6 +575,7 @@ impl LoadedScrollCaptureLiveTrace { } } +#[cfg(any(target_os = "macos", test))] fn resolve_trace_root_dir() -> Option { let override_dir = env::var_os(SCROLL_CAPTURE_TRACE_DIR_ENV).and_then(|value| { let trimmed = value.to_string_lossy().trim().to_owned(); @@ -583,6 +599,7 @@ fn resolve_trace_root_dir() -> Option { .map(|dirs| dirs.data_dir().join("scroll-capture-traces")) } +#[cfg(any(target_os = "macos", test))] fn parse_truthy_flag(value: &str) -> bool { let normalized = value.trim().to_ascii_lowercase(); @@ -593,6 +610,7 @@ fn duration_to_ms(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } +#[cfg(any(target_os = "macos", test))] fn now_unix_ms() -> Result { Ok(u64::try_from( SystemTime::now() From 8b58e127b8b613a8e0d6dadbce8195b4327c5646 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:32:39 +0800 Subject: [PATCH 08/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-ci","summary":"narrow live trace env entrypoints to macos","intent":"clear the remaining Linux Rust checks failure on PR #47","impact":"keep only macOS live-trace env plumbing on macOS while leaving replay and fixture helpers portable","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- .../rsnap-overlay/src/overlay/trace_recording.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index 5947650c..0e42d911 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -1,13 +1,11 @@ #[cfg(any(target_os = "macos", test))] use std::{ - env, - path::Path, - process, + env, process, time::{SystemTime, UNIX_EPOCH}, }; use std::{ fs, - path::PathBuf, + path::{Path, PathBuf}, time::{Duration, Instant}, }; @@ -24,9 +22,9 @@ use crate::{ scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}, }; -#[cfg(any(target_os = "macos", test))] +#[cfg(target_os = "macos")] const SCROLL_CAPTURE_TRACE_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE"; -#[cfg(any(target_os = "macos", test))] +#[cfg(target_os = "macos")] const SCROLL_CAPTURE_TRACE_DIR_ENV: &str = "RSNAP_SCROLL_CAPTURE_TRACE_DIR"; #[cfg(any(target_os = "macos", test))] const SCROLL_CAPTURE_TRACE_SCHEMA: &str = "scroll_capture_live_trace/1"; @@ -245,7 +243,7 @@ pub(crate) struct ScrollCaptureTraceRecorder { manifest: ScrollCaptureLiveTraceManifest, } impl ScrollCaptureTraceRecorder { - #[cfg(any(target_os = "macos", test))] + #[cfg(target_os = "macos")] pub(crate) fn from_env( monitor: MonitorRect, capture_rect_pixels: RectPoints, @@ -575,7 +573,7 @@ impl LoadedScrollCaptureLiveTrace { } } -#[cfg(any(target_os = "macos", test))] +#[cfg(target_os = "macos")] fn resolve_trace_root_dir() -> Option { let override_dir = env::var_os(SCROLL_CAPTURE_TRACE_DIR_ENV).and_then(|value| { let trimmed = value.to_string_lossy().trim().to_owned(); @@ -599,7 +597,7 @@ fn resolve_trace_root_dir() -> Option { .map(|dirs| dirs.data_dir().join("scroll-capture-traces")) } -#[cfg(any(target_os = "macos", test))] +#[cfg(target_os = "macos")] fn parse_truthy_flag(value: &str) -> bool { let normalized = value.trim().to_ascii_lowercase(); From 89bb5e86f1fc48fe3386e0acd192542e8406b12c Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:36:38 +0800 Subject: [PATCH 09/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-ci","summary":"gate remaining macos-only imports","intent":"clear the follow-up Linux Rust checks failure on PR #47","impact":"keep macOS-only imports out of Linux lint surfaces while preserving scroll-capture behavior","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- apps/rsnap/src/app/capture.rs | 1 + packages/rsnap-overlay/src/overlay/trace_recording.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/rsnap/src/app/capture.rs b/apps/rsnap/src/app/capture.rs index 549a53ba..1a0c1357 100644 --- a/apps/rsnap/src/app/capture.rs +++ b/apps/rsnap/src/app/capture.rs @@ -1,5 +1,6 @@ #[cfg(target_os = "macos")] use std::sync::Arc; +#[cfg(target_os = "macos")] use std::sync::atomic::Ordering; #[cfg(target_os = "macos")] use std::time::Duration; diff --git a/packages/rsnap-overlay/src/overlay/trace_recording.rs b/packages/rsnap-overlay/src/overlay/trace_recording.rs index 0e42d911..8b184785 100644 --- a/packages/rsnap-overlay/src/overlay/trace_recording.rs +++ b/packages/rsnap-overlay/src/overlay/trace_recording.rs @@ -10,7 +10,7 @@ use std::{ }; use color_eyre::eyre::{self, Result, WrapErr}; -#[cfg(any(target_os = "macos", test))] +#[cfg(target_os = "macos")] use directories::ProjectDirs; use image::RgbaImage; use serde::{Deserialize, Serialize}; From 872330fa89f9619348f6b65d265082ece0521f38 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 16:52:00 +0800 Subject: [PATCH 10/19] {"schema":"delivery/1","type":"fix","scope":"rust-style","summary":"move module docs inside modules","intent":"clear the remaining vstyle mod-doc violations on PR #47","impact":"satisfy Rust style checks without changing scroll-capture behavior","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- apps/rsnap/src/lib.rs | 1 - apps/rsnap/src/settings_window.rs | 3 ++- packages/rsnap-overlay/src/lib.rs | 6 ++++-- packages/rsnap-overlay/src/scroll_capture.rs | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/rsnap/src/lib.rs b/apps/rsnap/src/lib.rs index 10b5a888..63bf5927 100644 --- a/apps/rsnap/src/lib.rs +++ b/apps/rsnap/src/lib.rs @@ -2,7 +2,6 @@ #![allow(unused_crate_dependencies)] -/// Settings-window rendering and benchmark helpers used by benches and tests. pub mod settings_window; mod app; diff --git a/apps/rsnap/src/settings_window.rs b/apps/rsnap/src/settings_window.rs index 0f5edefd..ca0d12dc 100644 --- a/apps/rsnap/src/settings_window.rs +++ b/apps/rsnap/src/settings_window.rs @@ -1,4 +1,5 @@ -/// Deterministic settings-window benchmark helpers used by Criterion benches. +//! Settings-window rendering and benchmark helpers used by benches and tests. + pub mod bench_support; mod chrome; diff --git a/packages/rsnap-overlay/src/lib.rs b/packages/rsnap-overlay/src/lib.rs index 93efef2e..175c7dc0 100644 --- a/packages/rsnap-overlay/src/lib.rs +++ b/packages/rsnap-overlay/src/lib.rs @@ -5,15 +5,17 @@ #![allow(unused_crate_dependencies)] -/// Benchmark harness exports used by Criterion benches. pub mod bench_support { + //! Benchmark harness exports used by Criterion benches. + pub use crate::scroll_capture::bench_support::{ ScrollCaptureBenchHarness, ScrollCaptureBenchScenario, ScrollCaptureFingerprintMetrics, ScrollCaptureOverlapMetrics, ScrollCaptureSessionMetrics, }; } -/// Deterministic replay harness exports for scroll-capture verification. pub mod replay_support { + //! Deterministic replay harness exports for scroll-capture verification. + pub use crate::overlay::replay_support::{ RecordedScrollCaptureReplayMode, RecordedScrollCaptureReplayRecordedOutcome, RecordedScrollCaptureReplayStepResult, RecordedScrollCaptureReplaySummary, diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index e4beec9b..55a4d5ba 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -1,5 +1,6 @@ -/// Deterministic scroll-capture fixtures and harnesses used by Criterion benches. pub mod bench_support { + //! Deterministic scroll-capture fixtures and harnesses used by Criterion benches. + use image::{Rgba, RgbaImage, imageops}; use crate::scroll_capture::{ From 7fb4d5c7e9d42ef5549da925c4f4a1a4ac3286f6 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 18:54:27 +0800 Subject: [PATCH 11/19] {"schema":"delivery/1","type":"fix","scope":"replay-ci","summary":"stabilize replay self-check off macos","intent":"clear the remaining Rust checks failure on PR #47 under headless Linux","impact":"avoid headless cursor-device setup and keep recorded-source replay self-check macOS-only while preserving worker-pairwise coverage everywhere","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 14 ++++++++++++++ .../rsnap-overlay/src/overlay/replay_support.rs | 1 + 2 files changed, 15 insertions(+) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index fe2e7767..0c925c2b 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -10,6 +10,8 @@ mod window_runtime; #[cfg(target_os = "macos")] use std::collections::VecDeque; +#[cfg(not(target_os = "macos"))] +use std::env; #[cfg(target_os = "macos")] use std::ffi::c_void; use std::mem; @@ -747,6 +749,18 @@ pub struct OverlaySession { impl OverlaySession { #[cfg(not(target_os = "macos"))] fn try_create_cursor_device() -> Option { + let has_display = + env::var_os("DISPLAY").is_some() || env::var_os("WAYLAND_DISPLAY").is_some(); + + if !has_display { + tracing::warn!( + op = "overlay.cursor_device_unavailable", + "Skipping cursor-device initialization because no display server is available." + ); + + return None; + } + match panic::catch_unwind(device_query::DeviceState::new) { Ok(cursor_device) => Some(cursor_device), Err(_) => { diff --git a/packages/rsnap-overlay/src/overlay/replay_support.rs b/packages/rsnap-overlay/src/overlay/replay_support.rs index 47455aad..1649f20b 100644 --- a/packages/rsnap-overlay/src/overlay/replay_support.rs +++ b/packages/rsnap-overlay/src/overlay/replay_support.rs @@ -1170,6 +1170,7 @@ mod tests { image } + #[cfg(target_os = "macos")] #[test] fn replay_recorded_live_trace_round_trips_one_commit() { let rows = [ From 7923a4f77c1a25d2e313984f7c23e578268e3833 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 19:23:00 +0800 Subject: [PATCH 12/19] {"schema":"delivery/1","type":"fix","scope":"replay-ci","summary":"gate macos-only replay test helper","intent":"clear the remaining dead-code failure on PR #47","impact":"keep the tiny recorded-source helper aligned with the macOS-only replay self-check boundary","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay/replay_support.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rsnap-overlay/src/overlay/replay_support.rs b/packages/rsnap-overlay/src/overlay/replay_support.rs index 1649f20b..bfb494ff 100644 --- a/packages/rsnap-overlay/src/overlay/replay_support.rs +++ b/packages/rsnap-overlay/src/overlay/replay_support.rs @@ -1129,6 +1129,7 @@ mod tests { } } + #[cfg(target_os = "macos")] fn capture_rect() -> RectPoints { RectPoints::new(100, 120, 3, 5) } From 4a0d6d42f90db66f21f1f437fc566cc240b6a9a2 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 19:56:23 +0800 Subject: [PATCH 13/19] {"schema":"delivery/1","type":"fix","scope":"review-repair","summary":"repair review findings for worker-only scroll runtime","intent":"address the remaining PR feedback without reintroducing mixed live-stream capture","impact":"reduce hot-path tap logging and stop priming scroll live streams when worker pairwise sampling is authoritative","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- apps/rsnap/src/app/scroll_input_macos/tap.rs | 6 +- packages/rsnap-overlay/src/overlay.rs | 105 ++++++++++++++---- .../src/overlay/scroll_runtime.rs | 2 +- 3 files changed, 87 insertions(+), 26 deletions(-) diff --git a/apps/rsnap/src/app/scroll_input_macos/tap.rs b/apps/rsnap/src/app/scroll_input_macos/tap.rs index 0dede860..162ac16f 100644 --- a/apps/rsnap/src/app/scroll_input_macos/tap.rs +++ b/apps/rsnap/src/app/scroll_input_macos/tap.rs @@ -173,7 +173,7 @@ unsafe extern "C" fn scroll_input_event_tap_callback( fn send_overlay_scroll_input(context: &ScrollInputTapContext, cg_event: CGEventRef) { if !context.shared_state.is_enabled() { - tracing::info!( + tracing::debug!( op = "scroll_input.tap_ignored_disabled", "Discarded native scroll input event because scroll capture replay is disabled." ); @@ -182,7 +182,7 @@ fn send_overlay_scroll_input(context: &ScrollInputTapContext, cg_event: CGEventR } let Some(decoded) = decode::decode_scroll_input_from_cg_event(cg_event) else { - tracing::info!( + tracing::debug!( op = "scroll_input.tap_ignored_decode_none", "Discarded native scroll input event because it decoded to no usable delta." ); @@ -190,7 +190,7 @@ fn send_overlay_scroll_input(context: &ScrollInputTapContext, cg_event: CGEventR return; }; - tracing::info!( + tracing::debug!( op = "scroll_input.tap_forwarding", delta_y = decoded.delta_y, global_x = decoded.global_x, diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 0c925c2b..a2f1bdd5 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -1017,27 +1017,38 @@ impl OverlaySession { )); if self.scroll_capture.active { - self.scroll_capture.live_stream = match ( - self.scroll_capture.capture_rect_points, - self.scroll_capture.capture_rect_pixels, - ) { - (Some(capture_rect_points), Some(capture_rect_pixels)) => { - Some(MacLiveFrameStream::with_scroll_capture_region_and_waker( - self.config.self_capture_exception_window_ids.clone(), - capture_rect_points, - capture_rect_pixels, - self.scroll_frame_waker.clone(), - )) - }, - _ => Some(MacLiveFrameStream::with_self_capture_exception_window_ids_and_waker( - self.config.self_capture_exception_window_ids.clone(), - self.scroll_frame_waker.clone(), - )), + self.scroll_capture.live_stream = if self.should_use_scroll_capture_worker_sampling() { + None + } else { + match ( + self.scroll_capture.capture_rect_points, + self.scroll_capture.capture_rect_pixels, + ) { + (Some(capture_rect_points), Some(capture_rect_pixels)) => { + Some(MacLiveFrameStream::with_scroll_capture_region_and_waker( + self.config.self_capture_exception_window_ids.clone(), + capture_rect_points, + capture_rect_pixels, + self.scroll_frame_waker.clone(), + )) + }, + _ => { + Some(MacLiveFrameStream::with_self_capture_exception_window_ids_and_waker( + self.config.self_capture_exception_window_ids.clone(), + self.scroll_frame_waker.clone(), + )) + }, + } }; + + self.scroll_capture.live_stream_backlog.clear(); + self.scroll_capture.last_stream_frame_seq = 0; self.scroll_capture.last_stream_frame_fingerprint = None; self.scroll_capture.consecutive_identical_stream_frames = 0; self.scroll_capture.last_consumed_stream_frame_captured_at = None; + self.scroll_capture.last_stream_event_at = None; + self.scroll_capture.last_stream_poll_at = None; self.scroll_capture.pending_post_stall_burst_after_seq = None; self.scroll_capture.live_stream_stale_grace = None; self.scroll_capture.last_duplicate_stream_refresh_at = None; @@ -5827,6 +5838,7 @@ impl OverlaySession { capture_rect_pixels: RectPoints, base_frame: RgbaImage, ) -> Result { + let use_worker_sampling = self.should_use_scroll_capture_worker_sampling(); let trace_recorder = ScrollCaptureTraceRecorder::from_env( monitor, capture_rect_pixels, @@ -5864,12 +5876,14 @@ impl OverlaySession { #[cfg(target_os = "macos")] pixel_delta_residual: MacOSScrollPixelResidual::default(), #[cfg(target_os = "macos")] - live_stream: Some(MacLiveFrameStream::with_scroll_capture_region_and_waker( - self.config.self_capture_exception_window_ids.clone(), - capture_rect_points, - capture_rect_pixels, - self.scroll_frame_waker.clone(), - )), + live_stream: (!use_worker_sampling).then(|| { + MacLiveFrameStream::with_scroll_capture_region_and_waker( + self.config.self_capture_exception_window_ids.clone(), + capture_rect_points, + capture_rect_pixels, + self.scroll_frame_waker.clone(), + ) + }), #[cfg(target_os = "macos")] live_stream_backlog: VecDeque::new(), last_stream_frame_seq: 0, @@ -15223,6 +15237,22 @@ mod tests { assert!(session.scroll_capture.live_stream.is_some()); } + #[cfg(target_os = "macos")] + #[test] + fn scroll_capture_start_skips_scroll_live_stream_when_worker_sampling_is_forced() { + let mut session = OverlaySession::new(); + + seed_ready_scroll_capture_selection(&mut session); + enable_test_worker_scroll_capture_path(&mut session); + + let control = session.start_scroll_capture(); + + assert!(matches!(control, OverlayControl::Continue)); + assert!(session.scroll_capture.active); + assert!(session.scroll_capture.live_stream.is_none()); + assert!(session.scroll_capture.live_stream_backlog.is_empty()); + } + #[cfg(target_os = "macos")] #[test] fn reset_for_start_preserves_external_scroll_input_drain_reader() { @@ -18107,6 +18137,37 @@ mod tests { assert_eq!(session.scroll_capture.live_stream_stale_grace, None); } + #[cfg(target_os = "macos")] + #[test] + fn apply_self_capture_exception_window_ids_to_active_streams_keeps_scroll_live_stream_disabled_in_worker_mode() + { + let (mut session, original_worker_debug_id) = configured_session_with_macos_worker(); + + enable_test_worker_scroll_capture_path(&mut session); + + session.test_push_scroll_capture_live_frame(ScrollCaptureLiveFrame { + frame_seq: 9, + captured_at: Instant::now(), + image: test_frozen_image(), + }); + + session.scroll_capture.last_stream_event_at = Some(Instant::now()); + session.scroll_capture.last_stream_poll_at = Some(Instant::now()); + + session.apply_self_capture_exception_window_ids_to_active_streams(); + + assert_eq!( + session.live_sample_stream.as_ref().unwrap().debug_self_capture_exception_window_ids(), + &[17] + ); + assert!(session.scroll_capture.live_stream.is_none()); + assert!(session.scroll_capture.live_stream_backlog.is_empty()); + assert!(session.scroll_capture.last_stream_event_at.is_none()); + assert!(session.scroll_capture.last_stream_poll_at.is_none()); + assert_ne!(session.worker.as_ref().unwrap().debug_id(), original_worker_debug_id); + assert!(!session.pending_self_capture_exception_window_ids_worker_refresh); + } + #[cfg(target_os = "macos")] #[test] fn apply_self_capture_exception_window_ids_to_active_streams_defers_worker_refresh_while_freeze_is_inflight() diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index bdee0b05..baf44a56 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -33,7 +33,7 @@ use crate::worker::WorkerRequestSendError; impl OverlaySession { #[cfg(target_os = "macos")] - fn should_use_scroll_capture_worker_sampling(&self) -> bool { + pub(super) fn should_use_scroll_capture_worker_sampling(&self) -> bool { if !cfg!(test) { return true; } From bb1d1427c4c1293ca5d4801eaa8f130bea4cf227 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 2 Apr 2026 20:39:29 +0800 Subject: [PATCH 14/19] {"schema":"delivery/1","type":"fix","scope":"review-repair","summary":"repair scroll worker error handling review finding","intent":"prevent worker sample stalls after transient region-capture failures","impact":"clear scroll-capture inflight state and pause the session when CaptureMonitorRegion fails","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index a2f1bdd5..5b720a90 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -2681,7 +2681,12 @@ impl OverlaySession { self.png_encode_inflight = false; } }, - WorkerErrorSource::CaptureMonitorRegion => {}, + WorkerErrorSource::CaptureMonitorRegion => { + self.clear_scroll_capture_inflight_request(); + self.scroll_capture_set_error(message); + + return OverlayControl::Continue; + }, } self.state.set_error(message); @@ -18360,6 +18365,32 @@ mod tests { assert!(!session.pending_self_capture_exception_window_ids_worker_refresh); } + #[cfg(target_os = "macos")] + #[test] + fn capture_monitor_region_error_clears_scroll_capture_inflight_and_pauses_session() { + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.inflight_request_id = Some(41); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: true, + external_input_seq: 9, + input_direction: Some(ScrollDirection::Down), + }); + + let control = session.maybe_tick_worker_response_limiter(WorkerResponse::Error { + source: WorkerErrorSource::CaptureMonitorRegion, + message: String::from("capture timed out"), + }); + + assert!(matches!(control, super::OverlayControl::Continue)); + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert!(session.scroll_capture.paused); + assert_eq!(session.state.error_message.as_deref(), Some("capture timed out")); + } + #[test] fn downward_frame_motion_commits_even_with_legacy_upward_input_direction() { let document = [ From e1cc7602e7597a5a87a8dff332137efd7c963aa1 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 3 Apr 2026 08:04:54 +0800 Subject: [PATCH 15/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture","summary":"repair stale-input gate propagation and replay self-check isolation","intent":"fix external review finding and keep replay verification deterministic","impact":"passes stale-input override into the live-stream gate and avoids replay temp-trace collisions under cargo test concurrency","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 51 +++++++++++++++++++ .../src/overlay/replay_support.rs | 9 +++- .../src/overlay/scroll_runtime.rs | 2 +- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 5b720a90..1d01ae45 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -16215,6 +16215,57 @@ mod tests { assert_eq!(scroll_capture_export_height(&session), 6); } + #[cfg(target_os = "macos")] + #[test] + fn handle_scroll_capture_frame_passes_allow_stale_input_into_live_stream_gate() { + let document = [ + [10, 0, 0, 255], + [20, 0, 0, 255], + [30, 0, 0, 255], + [40, 0, 0, 255], + [50, 0, 0, 255], + [60, 0, 0, 255], + [70, 0, 0, 255], + [80, 0, 0, 255], + ]; + let monitor = MonitorRect { + id: 1, + origin: GlobalPoint::new(0, 0), + width: 1_000, + height: 800, + scale_factor_x1000: 1_000, + }; + let capture_rect = RectPoints::new(100, 120, 200, 240); + let observed_at = Instant::now(); + let input_at = observed_at - SCROLL_CAPTURE_INPUT_FRESHNESS - Duration::from_millis(1); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.input_direction = Some(ScrollDirection::Down); + session.scroll_capture.input_direction_at = Some(input_at); + session.scroll_capture.session = + Some(ScrollSession::new(make_scroll_capture_window(&document, 3, 0, 5), 320).unwrap()); + + assert_eq!( + session + .handle_scroll_capture_frame( + make_scroll_capture_window(&document, 3, 1, 5), + ScrollCaptureFrameSource::LiveStream { frame_seq: 143 }, + true, + observed_at, + ) + .transpose() + .unwrap(), + Some(ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: 1, + }) + ); + assert_eq!(scroll_capture_export_height(&session), 6); + } + #[cfg(target_os = "macos")] #[test] fn fresh_live_stream_frame_without_direction_metadata_fails_closed_as_no_change() { diff --git a/packages/rsnap-overlay/src/overlay/replay_support.rs b/packages/rsnap-overlay/src/overlay/replay_support.rs index bfb494ff..a5959dd9 100644 --- a/packages/rsnap-overlay/src/overlay/replay_support.rs +++ b/packages/rsnap-overlay/src/overlay/replay_support.rs @@ -1093,6 +1093,7 @@ mod tests { fs, path::PathBuf, process, + sync::atomic::{AtomicU64, Ordering}, time::{Duration, Instant}, }; @@ -1108,11 +1109,15 @@ mod tests { }; use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; + static TRACE_TEST_ROOT_COUNTER: AtomicU64 = AtomicU64::new(0); + fn temp_trace_root() -> PathBuf { + let counter = TRACE_TEST_ROOT_COUNTER.fetch_add(1, Ordering::Relaxed); let root = env::temp_dir().join(format!( - "rsnap-recorded-trace-replay-test-{}-{}", + "rsnap-recorded-trace-replay-test-{}-{}-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis(), - process::id() + process::id(), + counter )); let _ = fs::remove_dir_all(&root); diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index baf44a56..099b8d1e 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -926,7 +926,7 @@ impl OverlaySession { } else { self.observe_scroll_capture_frame_with_gate( frame, - false, + allow_stale_input, observation_at, allow_post_stall_burst_search, )? From a8768c8042d67f66095cc9b46d78574b5ee6dcb1 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 3 Apr 2026 08:39:21 +0800 Subject: [PATCH 16/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-review-repair","summary":"repair PR review fixes for screenshot fallback and worker input gating","intent":"keep macOS screenshot capture compatible with documented support while fail-closing ungated worker frames","impact":"restores pre-Sonoma region-capture fallback and prevents worker pairwise commits without fresh or latched downward input","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/backend.rs | 55 ++++++++++++++++++- packages/rsnap-overlay/src/overlay.rs | 40 ++++++++++++++ .../src/overlay/scroll_runtime.rs | 14 +++-- 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index 99a88264..1b82aafc 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -32,7 +32,7 @@ use objc2_core_graphics::{ CGDataProvider, CGImage, CGRectNull, CGWindowID, CGWindowImageOption, CGWindowListOption, }; #[cfg(target_os = "macos")] -use objc2_foundation::NSError; +use objc2_foundation::{NSError, NSOperatingSystemVersion, NSProcessInfo}; #[cfg(target_os = "macos")] use objc2_screen_capture_kit::SCScreenshotManager; use thiserror::Error; @@ -521,6 +521,26 @@ impl XcapCaptureBackend { monitor: MonitorRect, rect_px: RectPoints, ) -> Result> { + if !Self::macos_supports_scroll_capture_screenshot_api() { + let image = capture_monitor_region_with_core_graphics(monitor, rect_px).wrap_err_with( + || { + format!( + "failed to capture monitor region via CoreGraphics fallback: {monitor:?}" + ) + }, + )?; + + tracing::trace!( + op = "capture_backend.region_core_graphics_fallback", + monitor_id = monitor.id, + rect_px = ?rect_px, + frame_px = ?image.dimensions(), + "Captured monitor region from the CoreGraphics fallback because the screenshot API is unavailable." + ); + + return Ok(Some(image)); + } + let image = capture_monitor_region_image_with_screenshot_manager(monitor, rect_px) .wrap_err_with(|| { format!("failed to capture monitor region via SCScreenshotManager: {monitor:?}") @@ -537,6 +557,15 @@ impl XcapCaptureBackend { Ok(Some(image)) } + #[cfg(target_os = "macos")] + fn macos_supports_scroll_capture_screenshot_api() -> bool { + let process_info = NSProcessInfo::processInfo(); + + macos_supports_scroll_capture_screenshot_api_with_version( + process_info.operatingSystemVersion(), + ) + } + #[cfg(target_os = "macos")] fn region_capture_after_seq(&self, monitor: MonitorRect, rect_px: RectPoints) -> u64 { self.last_region_capture @@ -1071,6 +1100,13 @@ fn copy_rgba_patch( out } +#[cfg(target_os = "macos")] +fn macos_supports_scroll_capture_screenshot_api_with_version( + version: NSOperatingSystemVersion, +) -> bool { + version.majorVersion >= 14 +} + fn normalize_capture_rect(rect_px: RectPoints) -> RectPoints { RectPoints::new(rect_px.x, rect_px.y, rect_px.width.max(1), rect_px.height.max(1)) } @@ -1437,6 +1473,9 @@ fn xcap_find_monitor(monitor: MonitorRect) -> Result { #[cfg(test)] mod tests { + #[cfg(target_os = "macos")] + use objc2_foundation::NSOperatingSystemVersion; + #[cfg(target_os = "macos")] use crate::backend::{self}; use crate::backend::{CaptureBackend, StubCaptureBackend}; @@ -1511,4 +1550,18 @@ mod tests { assert!(format!("{err:#}").contains("shorter than the declared image size")); } + + #[cfg(target_os = "macos")] + #[test] + fn screenshot_api_scroll_capture_gate_requires_macos_14_or_newer() { + assert!(!backend::macos_supports_scroll_capture_screenshot_api_with_version( + NSOperatingSystemVersion { majorVersion: 13, minorVersion: 6, patchVersion: 0 } + )); + assert!(backend::macos_supports_scroll_capture_screenshot_api_with_version( + NSOperatingSystemVersion { majorVersion: 14, minorVersion: 0, patchVersion: 0 } + )); + assert!(backend::macos_supports_scroll_capture_screenshot_api_with_version( + NSOperatingSystemVersion { majorVersion: 15, minorVersion: 0, patchVersion: 0 } + )); + } } diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index 1d01ae45..b540f532 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -16984,6 +16984,46 @@ mod tests { assert_eq!(session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), 90); } + #[cfg(target_os = "macos")] + #[test] + fn worker_frame_without_fresh_or_latched_input_fails_closed_without_appending_growth() { + let monitor = test_monitor(); + let capture_rect = RectPoints::new(100, 120, 512, 640); + let base = make_sparse_worker_capture_window(512, 640, 0); + let next = make_sparse_worker_capture_window(512, 640, 90); + let mut session = OverlaySession::new(); + + session.scroll_capture.active = true; + session.scroll_capture.monitor = Some(monitor); + session.scroll_capture.capture_rect_pixels = Some(capture_rect); + session.scroll_capture.session = Some(ScrollSession::new(base, 320).unwrap()); + session.scroll_capture.inflight_request_id = Some(41); + session.scroll_capture.inflight_request_observation = + Some(InflightScrollCaptureObservation { + was_observable: false, + external_input_seq: 7, + input_direction: Some(ScrollDirection::Down), + }); + + let export_height_before = + session.scroll_capture.session.as_ref().unwrap().export_image().height(); + let viewport_top_before = + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(); + + session.handle_captured_scroll_region(monitor, capture_rect, 41, next); + + assert_eq!(session.scroll_capture.inflight_request_id, None); + assert_eq!(session.scroll_capture.inflight_request_observation, None); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().export_image().height(), + export_height_before + ); + assert_eq!( + session.scroll_capture.session.as_ref().unwrap().current_viewport_top_y(), + viewport_top_before + ); + } + #[cfg(target_os = "macos")] #[test] fn newer_opposite_direction_supersedes_latched_worker_observation_context() { diff --git a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs index 099b8d1e..c4504907 100644 --- a/packages/rsnap-overlay/src/overlay/scroll_runtime.rs +++ b/packages/rsnap-overlay/src/overlay/scroll_runtime.rs @@ -916,13 +916,17 @@ impl OverlaySession { let worker_pairwise_path = cfg!(target_os = "macos") && matches!(source, ScrollCaptureFrameSource::Worker { .. }); let outcome = if worker_pairwise_path { - let Some(session) = self.scroll_capture.session.as_mut() else { - self.scroll_capture_set_error("Scroll capture session is unavailable."); + if !allow_stale_input && prior_block_reason.is_some() { + Ok(ScrollObserveOutcome::NoChange) + } else { + let Some(session) = self.scroll_capture.session.as_mut() else { + self.scroll_capture_set_error("Scroll capture session is unavailable."); - return None; - }; + return None; + }; - session.observe_worker_pairwise_vision_frame(frame) + session.observe_worker_pairwise_vision_frame(frame) + } } else { self.observe_scroll_capture_frame_with_gate( frame, From 9706217baa990b2d4bfb2bf9ac74cb12f7984f6d Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 3 Apr 2026 09:00:06 +0800 Subject: [PATCH 17/19] {"schema":"delivery/1","type":"fix","scope":"scroll-capture-review-repair","summary":"tolerate undersized screenshot-manager frames in macOS scroll capture","intent":"keep rounded screenshot-manager captures from stalling worker sampling on stable-size paths","impact":"normalizes smaller screenshot-manager frames back to the requested extent so scroll capture avoids transient worker errors from inward-rounded captures","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/backend.rs | 50 +++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index 1b82aafc..391f3d51 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -1130,6 +1130,31 @@ fn crop_monitor_image_region(image: &RgbaImage, rect_px: RectPoints) -> Result RgbaImage { + let width = width.max(1); + let height = height.max(1); + + if image.dimensions() == (width, height) { + return image.clone(); + } + + let source_max_x = image.width().saturating_sub(1); + let source_max_y = image.height().saturating_sub(1); + let mut out = RgbaImage::new(width, height); + + for out_y in 0..height { + let sample_y = out_y.min(source_max_y); + + for out_x in 0..width { + let sample_x = out_x.min(source_max_x); + + out.put_pixel(out_x, out_y, *image.get_pixel(sample_x, sample_y)); + } + } + + out +} + #[cfg(target_os = "macos")] fn capture_monitor_region_image_with_screenshot_manager( monitor: MonitorRect, @@ -1148,11 +1173,11 @@ fn capture_monitor_region_image_with_screenshot_manager( let image = rgba_image_from_cg_image(cg_image.as_ref())?; // ScreenCaptureKit may round point-space captures by one pixel at non-integer scale edges. - // Clamp back to the requested region so the stitcher sees stable dimensions. + // Clamp or extend back to the requested region so the stitcher sees stable dimensions. if image.dimensions() == (rect_px.width, rect_px.height) { Ok(image) } else { - crop_monitor_image_region(&image, RectPoints::new(0, 0, rect_px.width, rect_px.height)) + Ok(normalize_capture_image_extent(&image, rect_px.width, rect_px.height)) } } @@ -1473,6 +1498,7 @@ fn xcap_find_monitor(monitor: MonitorRect) -> Result { #[cfg(test)] mod tests { + use image::RgbaImage; #[cfg(target_os = "macos")] use objc2_foundation::NSOperatingSystemVersion; @@ -1564,4 +1590,24 @@ mod tests { NSOperatingSystemVersion { majorVersion: 15, minorVersion: 0, patchVersion: 0 } )); } + + #[test] + fn normalize_capture_image_extent_pads_inward_rounded_edges_with_border_pixels() { + let image = RgbaImage::from_vec( + 2, + 2, + vec![ + 10, 0, 0, 255, 20, 0, 0, 255, // + 30, 0, 0, 255, 40, 0, 0, 255, + ], + ) + .expect("valid rgba image"); + let normalized = backend::normalize_capture_image_extent(&image, 3, 3); + + assert_eq!(normalized.dimensions(), (3, 3)); + assert_eq!(normalized.get_pixel(0, 0).0, [10, 0, 0, 255]); + assert_eq!(normalized.get_pixel(2, 0).0, [20, 0, 0, 255]); + assert_eq!(normalized.get_pixel(0, 2).0, [30, 0, 0, 255]); + assert_eq!(normalized.get_pixel(2, 2).0, [40, 0, 0, 255]); + } } From 8ea3c43dd96d53480e2a9a9ee79f76d53c2be423 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 3 Apr 2026 09:11:38 +0800 Subject: [PATCH 18/19] {"schema":"delivery/1","type":"fix","scope":"ci-review-repair","summary":"restore backend test import for Linux rust checks","intent":"repair the backend test module so the new screenshot normalization regression test compiles on non-macOS CI","impact":"keeps the undersized screenshot-frame fix while restoring cross-platform Rust checks for PR #47","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/backend.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/rsnap-overlay/src/backend.rs b/packages/rsnap-overlay/src/backend.rs index 391f3d51..b30e0920 100644 --- a/packages/rsnap-overlay/src/backend.rs +++ b/packages/rsnap-overlay/src/backend.rs @@ -1502,8 +1502,7 @@ mod tests { #[cfg(target_os = "macos")] use objc2_foundation::NSOperatingSystemVersion; - #[cfg(target_os = "macos")] - use crate::backend::{self}; + use crate::backend; use crate::backend::{CaptureBackend, StubCaptureBackend}; #[cfg(target_os = "macos")] use crate::state::{GlobalPoint, MonitorRect, RectPoints}; From 9e13c4ed1e1d12b0a0b35f597aff5ed189175cc5 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 3 Apr 2026 09:32:14 +0800 Subject: [PATCH 19/19] {"schema":"delivery/1","type":"fix","scope":"review-repair","summary":"restore non-macOS worker scroll sampling cadence","intent":"keep the macOS screenshot worker on the slower 250ms cadence while restoring the tighter non-macOS polling interval expected by the worker-only path","impact":"addresses PR review feedback by preventing Linux and Windows worker sampling from regressing to the slower macOS cadence","breaking":false,"risk":"low","authority":"linear","delivery_mode":"status-only","refs":[{"system":"linear","id":"XY-185","role":"authority"},{"system":"github","repo":"hack-ink/rsnap","number":47,"role":"mirror"}]} --- packages/rsnap-overlay/src/overlay.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/rsnap-overlay/src/overlay.rs b/packages/rsnap-overlay/src/overlay.rs index b540f532..b94eaaad 100644 --- a/packages/rsnap-overlay/src/overlay.rs +++ b/packages/rsnap-overlay/src/overlay.rs @@ -308,7 +308,10 @@ const WINDOW_CAPTURE_MATTE_DARK_RGBA: image::Rgba = image::Rgba([24, 24, 24, const SCROLL_PREVIEW_WINDOW_WIDTH_POINTS: f64 = 260.0; const SCROLL_PREVIEW_WINDOW_HEIGHT_POINTS: f64 = 360.0; const SCROLL_PREVIEW_WINDOW_MARGIN_POINTS: i32 = 16; +#[cfg(target_os = "macos")] const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); +#[cfg(not(target_os = "macos"))] +const SCROLL_CAPTURE_SAMPLE_INTERVAL: Duration = Duration::from_millis(50); #[cfg(target_os = "macos")] const SCROLL_CAPTURE_DUPLICATE_WORKER_FRAME_RETRY_INTERVAL: Duration = Duration::from_millis(60); #[cfg(target_os = "macos")] @@ -13016,7 +13019,6 @@ mod tests { use std::sync::Arc; #[cfg(target_os = "macos")] use std::thread; - #[cfg(target_os = "macos")] use std::time::Duration; use std::time::Instant; @@ -13049,12 +13051,13 @@ mod tests { use crate::overlay::{ FrozenSelectionDragState, FrozenToolbarState, FrozenToolbarTool, HUD_LOUPE_STRIP_GAP_POINTS, HudTheme, OverlaySession, Pos2, Rect, - SELECTION_DASHED_BORDER_DASH_LENGTH_PX, SELECTION_DASHED_BORDER_GAP_LENGTH_PX, - SELECTION_DASHED_BORDER_WIDTH_PX, SELECTION_SIZE_BADGE_GAP_PX, - SELECTION_SIZE_BADGE_INSIDE_MARGIN_PX, SELECTION_SIZE_BADGE_SCREEN_MARGIN_PX, - SelectionDashedBorderCache, SelectionDashedBorderMetrics, SelectionFlowGeometryCache, - SelectionSizeBadgeTarget, TOOLBAR_CAPTURE_GAP_PX, TOOLBAR_SCREEN_MARGIN_PX, - ToolbarPlacement, Vec2, WindowRenderer, hud_helpers, regular, + SCROLL_CAPTURE_SAMPLE_INTERVAL, SELECTION_DASHED_BORDER_DASH_LENGTH_PX, + SELECTION_DASHED_BORDER_GAP_LENGTH_PX, SELECTION_DASHED_BORDER_WIDTH_PX, + SELECTION_SIZE_BADGE_GAP_PX, SELECTION_SIZE_BADGE_INSIDE_MARGIN_PX, + SELECTION_SIZE_BADGE_SCREEN_MARGIN_PX, SelectionDashedBorderCache, + SelectionDashedBorderMetrics, SelectionFlowGeometryCache, SelectionSizeBadgeTarget, + TOOLBAR_CAPTURE_GAP_PX, TOOLBAR_SCREEN_MARGIN_PX, ToolbarPlacement, Vec2, WindowRenderer, + hud_helpers, regular, }; use crate::scroll_capture::{ScrollDirection, ScrollObserveOutcome, ScrollSession}; #[cfg(target_os = "macos")] @@ -17804,6 +17807,14 @@ mod tests { assert_eq!(scroll_capture_export_height(&session), 724); } + #[test] + fn scroll_capture_sample_interval_matches_platform_worker_sampling_strategy() { + #[cfg(target_os = "macos")] + assert_eq!(SCROLL_CAPTURE_SAMPLE_INTERVAL, Duration::from_millis(250)); + #[cfg(not(target_os = "macos"))] + assert_eq!(SCROLL_CAPTURE_SAMPLE_INTERVAL, Duration::from_millis(50)); + } + #[cfg(target_os = "macos")] #[test] fn maybe_tick_scroll_capture_worker_path_backs_off_after_duplicate_committed_frame() {