From 398a33ea31e332fa1cb11fd0f02252dfedcd4822 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 12 May 2026 00:04:09 +0800 Subject: [PATCH 1/3] {"schema":"decodex/commit/1","summary":"Refactor scroll capture cleanup boundary","authority":"manual"} --- docs/reference/host-core-reset.md | 4 + docs/reference/workspace-layout.md | 8 + ...ptureSessionController+ScrollCapture.swift | 163 +----- ...tiveScrollCaptureObservationPipeline.swift | 168 +++++++ packages/rsnap-overlay/src/scroll_capture.rs | 472 +----------------- .../src/scroll_capture/support.rs | 46 +- .../rsnap-overlay/src/scroll_capture/tests.rs | 109 ++-- .../src/scroll_capture/worker_pairwise.rs | 440 ++++++++++++++++ scripts/smoke/native-scroll-capture-macos.sh | 29 ++ 9 files changed, 753 insertions(+), 686 deletions(-) create mode 100644 native/macos-host/Sources/RsnapNativeHostKit/NativeScrollCaptureObservationPipeline.swift create mode 100644 packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs diff --git a/docs/reference/host-core-reset.md b/docs/reference/host-core-reset.md index 8fa698b1..3dd78046 100644 --- a/docs/reference/host-core-reset.md +++ b/docs/reference/host-core-reset.md @@ -72,6 +72,10 @@ The current native-host Swift split is: lifecycle hooks; the extensions split live capture/input, frozen selection interactions, host-request draining, native scroll-capture sampling, copy/save/export effects, Vision OCR, and runtime teardown/window helpers. +- `NativeScrollCaptureObservationPipeline.swift`: native scroll-capture sample batching, fallback + sample adaptation, Rust scroll-observation calls, and preview export refresh packaging. It keeps + ordered frame acquisition and AppKit scheduling in Swift while leaving stitching decisions in + Rust. - `CaptureChrome.swift`: shared native chrome metrics, palette, dashed-border geometry, and AppKit color/image helpers used by live and frozen capture UI. - `CaptureOverlayWindow.swift`: the AppKit `NSPanel` wrapper that embeds `CaptureHostView` for each diff --git a/docs/reference/workspace-layout.md b/docs/reference/workspace-layout.md index 98d2af4e..026ea97c 100644 --- a/docs/reference/workspace-layout.md +++ b/docs/reference/workspace-layout.md @@ -94,6 +94,12 @@ Key paths: - `packages/rsnap-overlay/src/live_frame_stream_macos.rs`: current macOS live-stream support - `packages/rsnap-overlay/src/scroll_capture.rs`: current scroll-capture session entry with focused support modules under `scroll_capture/` +- `packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs`: ordered worker-pairwise frame + registration, committed-frontier catchup, rewind/reacquire handling, and growth-block decisions +- `packages/rsnap-overlay/src/scroll_capture/support.rs`: shared pixel matching, fingerprinting, + static-region rejection, and image-analysis helpers used by scroll capture +- `packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs`: downward viewport candidate + scoring and resolution helpers for session-owned stitching decisions ### `packages/rsnap-capture-core/` @@ -165,6 +171,8 @@ The main host-kit files are split by responsibility: commit handling, host-owned frozen scene preparation, and host-effect dispatch - `CaptureSessionController+ScrollCapture.swift`: native scroll monitor lifecycle, scroll-event forwarding, viewport sampling, and scroll minimap preview refresh +- `NativeScrollCaptureObservationPipeline.swift`: conversion of ordered native samples and + fallback frames into Rust scroll observations plus preview export batches - `CaptureSessionController+Export.swift`: copy/save host effects, output naming, capture-image export, capture-frame effect application, and Rust-backed PNG encoding - `CaptureSessionController+TextRecognition.swift`: Vision OCR request execution and recognized diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift index dfb2be28..d4c59740 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+ScrollCapture.swift @@ -37,61 +37,8 @@ private func scrollCaptureViewportPointCandidates( return [point, flippedPoint] } -private struct NativeScrollCaptureSampleFrame: Sendable { - let region: RGBARegionSnapshot - let source: String - let frameSequence: UInt64 - let frameAgeMicroseconds: UInt64 -} - -private struct NativeScrollCaptureFallbackRequest: Sendable { - let rect: CGRect - let source: CaptureSessionController.FrozenCaptureJobSource - let frameSequence: UInt64 -} - -private struct NativeScrollCaptureObservation: Sendable { - let sampledFrame: NativeScrollCaptureSampleFrame - let registrationStrategy: String - let result: ScrollObserveResult? - let errorDescription: String? -} - private let nativeScrollCaptureMinimumNonzeroWheelMotionHintRows = 12.0 -private struct NativeScrollCapturePreviewUpdate: @unchecked Sendable { - let image: CGImage - let exportWidth: Int - let exportHeight: Int - let result: ScrollObserveResult - let viewportTopYPixels: Int - let viewportHeightPixels: Int -} - -private struct NativeScrollCaptureObservationBatch: Sendable { - let observations: [NativeScrollCaptureObservation] - let preview: NativeScrollCapturePreviewUpdate? - let previewErrorDescription: String? - let previewExportMilliseconds: Double? -} - -private func writeNativeScrollCaptureDebugDump(_ snapshot: RGBARegionSnapshot, name: String) { - guard - let rawDirectory = ProcessInfo.processInfo.environment["RSNAP_SCROLL_CAPTURE_DUMP_DIR"], - rawDirectory.isEmpty == false, - let pngData = try? RsnapExportEncoder.pngData(from: snapshot) - else { - return - } - let directory = URL(fileURLWithPath: rawDirectory, isDirectory: true) - try? FileManager.default.createDirectory( - at: directory, - withIntermediateDirectories: true - ) - let safeName = name.replacingOccurrences(of: "/", with: "_") - try? pngData.write(to: directory.appendingPathComponent("\(safeName).png")) -} - extension CaptureSessionController { private static let scrollCaptureForwardedEventMarker: Int64 = 0x5253_4E41_5053_4352 private static let scrollCapturePreciseWheelDeltaLimit = 72.0 @@ -745,7 +692,7 @@ extension CaptureSessionController { frameAgeMicroseconds: 0 )) } - let batch = Self.nativeScrollCaptureObservationBatch( + let batch = NativeScrollCaptureObservationPipeline.makeBatch( sampledFrames: sampledFrames, stitcher: stitcher, motionRowsHint: motionRowsHint, @@ -763,114 +710,6 @@ extension CaptureSessionController { } } - nonisolated private static func nativeScrollCaptureObservationBatch( - sampledFrames: [NativeScrollCaptureSampleFrame], - stitcher: RsnapScrollCaptureSession, - motionRowsHint: Int?, - previewRefreshDue: Bool - ) -> NativeScrollCaptureObservationBatch { - var observations: [NativeScrollCaptureObservation] = [] - var latestPreviewCandidate: NativeScrollCaptureObservation? - for sampledFrame in sampledFrames { - let observation = nativeScrollCaptureObservation( - sampledFrame, - stitcher: stitcher, - motionRowsHint: motionRowsHint - ) - if observation.result?.outcome != .noChange { - latestPreviewCandidate = observation - } - observations.append(observation) - } - let preview = nativeScrollCapturePreviewUpdate( - stitcher: stitcher, - candidate: latestPreviewCandidate, - previewRefreshDue: previewRefreshDue - ) - return NativeScrollCaptureObservationBatch( - observations: observations, - preview: preview.update, - previewErrorDescription: preview.errorDescription, - previewExportMilliseconds: preview.exportMilliseconds - ) - } - - nonisolated private static func nativeScrollCaptureObservation( - _ sampledFrame: NativeScrollCaptureSampleFrame, - stitcher: RsnapScrollCaptureSession, - motionRowsHint: Int? - ) -> NativeScrollCaptureObservation { - let registrationStrategy = "pairwise" - do { - let result = try stitcher.observeDownwardFrame( - sampledFrame.region, - motionRowsHint: motionRowsHint - ) - return NativeScrollCaptureObservation( - sampledFrame: sampledFrame, - registrationStrategy: registrationStrategy, - result: result, - errorDescription: nil - ) - } catch { - return NativeScrollCaptureObservation( - sampledFrame: sampledFrame, - registrationStrategy: registrationStrategy, - result: nil, - errorDescription: String(describing: error) - ) - } - } - - nonisolated private static func nativeScrollCapturePreviewUpdate( - stitcher: RsnapScrollCaptureSession, - candidate: NativeScrollCaptureObservation?, - previewRefreshDue: Bool - ) -> ( - update: NativeScrollCapturePreviewUpdate?, - errorDescription: String?, - exportMilliseconds: Double? - ) { - guard previewRefreshDue, let candidate, let result = candidate.result else { - return (nil, nil, nil) - } - let previewStartedAt = ProcessInfo.processInfo.systemUptime - do { - if let export = try stitcher.exportImage() { - guard let exportImage = NativeHostImageBridge.cgImage(from: export) else { - return ( - nil, - "scroll preview export returned no image", - NativeHostTelemetry.milliseconds(since: previewStartedAt) - ) - } - return ( - NativeScrollCapturePreviewUpdate( - image: exportImage, - exportWidth: export.width, - exportHeight: export.height, - result: result, - viewportTopYPixels: result.currentViewportTopY, - viewportHeightPixels: candidate.sampledFrame.region.height - ), - nil, - NativeHostTelemetry.milliseconds(since: previewStartedAt) - ) - } - return ( - nil, - "scroll preview export returned no image", - NativeHostTelemetry.milliseconds(since: previewStartedAt) - ) - } catch { - return ( - nil, - String(describing: error), - NativeHostTelemetry.milliseconds(since: previewStartedAt) - ) - } - } - private func finishNativeScrollCaptureObservations( _ batch: NativeScrollCaptureObservationBatch, captureID: UInt64, diff --git a/native/macos-host/Sources/RsnapNativeHostKit/NativeScrollCaptureObservationPipeline.swift b/native/macos-host/Sources/RsnapNativeHostKit/NativeScrollCaptureObservationPipeline.swift new file mode 100644 index 00000000..ca0c3c10 --- /dev/null +++ b/native/macos-host/Sources/RsnapNativeHostKit/NativeScrollCaptureObservationPipeline.swift @@ -0,0 +1,168 @@ +import AppKit +@preconcurrency import CoreGraphics +import Foundation +import RsnapHostBridge + +struct NativeScrollCaptureSampleFrame: Sendable { + let region: RGBARegionSnapshot + let source: String + let frameSequence: UInt64 + let frameAgeMicroseconds: UInt64 +} + +struct NativeScrollCaptureFallbackRequest: Sendable { + let rect: CGRect + let source: CaptureSessionController.FrozenCaptureJobSource + let frameSequence: UInt64 +} + +struct NativeScrollCaptureObservation: Sendable { + let sampledFrame: NativeScrollCaptureSampleFrame + let registrationStrategy: String + let result: ScrollObserveResult? + let errorDescription: String? +} + +struct NativeScrollCapturePreviewUpdate: @unchecked Sendable { + let image: CGImage + let exportWidth: Int + let exportHeight: Int + let result: ScrollObserveResult + let viewportTopYPixels: Int + let viewportHeightPixels: Int +} + +struct NativeScrollCaptureObservationBatch: Sendable { + let observations: [NativeScrollCaptureObservation] + let preview: NativeScrollCapturePreviewUpdate? + let previewErrorDescription: String? + let previewExportMilliseconds: Double? +} + +enum NativeScrollCaptureObservationPipeline { + static func makeBatch( + sampledFrames: [NativeScrollCaptureSampleFrame], + stitcher: RsnapScrollCaptureSession, + motionRowsHint: Int?, + previewRefreshDue: Bool + ) -> NativeScrollCaptureObservationBatch { + var observations: [NativeScrollCaptureObservation] = [] + var latestPreviewCandidate: NativeScrollCaptureObservation? + for sampledFrame in sampledFrames { + let observation = observe( + sampledFrame, + stitcher: stitcher, + motionRowsHint: motionRowsHint + ) + if observation.result?.outcome != .noChange { + latestPreviewCandidate = observation + } + observations.append(observation) + } + let preview = previewUpdate( + stitcher: stitcher, + candidate: latestPreviewCandidate, + previewRefreshDue: previewRefreshDue + ) + return NativeScrollCaptureObservationBatch( + observations: observations, + preview: preview.update, + previewErrorDescription: preview.errorDescription, + previewExportMilliseconds: preview.exportMilliseconds + ) + } + + private static func observe( + _ sampledFrame: NativeScrollCaptureSampleFrame, + stitcher: RsnapScrollCaptureSession, + motionRowsHint: Int? + ) -> NativeScrollCaptureObservation { + let registrationStrategy = "pairwise" + do { + let result = try stitcher.observeDownwardFrame( + sampledFrame.region, + motionRowsHint: motionRowsHint + ) + return NativeScrollCaptureObservation( + sampledFrame: sampledFrame, + registrationStrategy: registrationStrategy, + result: result, + errorDescription: nil + ) + } catch { + return NativeScrollCaptureObservation( + sampledFrame: sampledFrame, + registrationStrategy: registrationStrategy, + result: nil, + errorDescription: String(describing: error) + ) + } + } + + private static func previewUpdate( + stitcher: RsnapScrollCaptureSession, + candidate: NativeScrollCaptureObservation?, + previewRefreshDue: Bool + ) -> ( + update: NativeScrollCapturePreviewUpdate?, + errorDescription: String?, + exportMilliseconds: Double? + ) { + guard previewRefreshDue, let candidate, let result = candidate.result else { + return (nil, nil, nil) + } + let previewStartedAt = ProcessInfo.processInfo.systemUptime + do { + if let export = try stitcher.exportImage() { + guard let exportImage = NativeHostImageBridge.cgImage(from: export) else { + return ( + nil, + "scroll preview export returned no image", + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } + + return ( + NativeScrollCapturePreviewUpdate( + image: exportImage, + exportWidth: export.width, + exportHeight: export.height, + result: result, + viewportTopYPixels: result.currentViewportTopY, + viewportHeightPixels: candidate.sampledFrame.region.height + ), + nil, + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } + return ( + nil, + "scroll preview export returned no image", + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } catch { + return ( + nil, + String(describing: error), + NativeHostTelemetry.milliseconds(since: previewStartedAt) + ) + } + } +} + +func writeNativeScrollCaptureDebugDump(_ snapshot: RGBARegionSnapshot, name: String) { + guard + let rawDirectory = ProcessInfo.processInfo.environment["RSNAP_SCROLL_CAPTURE_DUMP_DIR"], + rawDirectory.isEmpty == false, + let pngData = try? RsnapExportEncoder.pngData(from: snapshot) + else { + return + } + let directory = URL(fileURLWithPath: rawDirectory, isDirectory: true) + try? FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true + ) + let safeName = name.replacingOccurrences(of: "/", with: "_") + try? pngData.write(to: directory.appendingPathComponent("\(safeName).png")) +} diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index 641c1ad7..edfac02d 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -2,6 +2,7 @@ pub mod bench_support; mod downward_resolution; mod support; +mod worker_pairwise; pub(crate) use self::support::{ compose_provisional_preview_image, scroll_capture_fingerprint, scroll_capture_fingerprint_delta, @@ -9,7 +10,7 @@ pub(crate) use self::support::{ use std::ops::RangeInclusive; -use color_eyre::eyre::{self}; +use color_eyre::eyre::{self, Result}; use image::RgbaImage; #[cfg(test)] @@ -45,8 +46,6 @@ const FALLBACK_DOWNWARD_GROWTH_MIN_ROWS: u32 = 8; const FALLBACK_DOWNWARD_GROWTH_MAX_ROWS: u32 = 16; const TRANSIENT_MOTION_HINT_MAX_MULTIPLIER: u32 = 3; const TRANSIENT_MOTION_HINT_MIN_CAP_ROWS: u32 = 12; -const WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS: u32 = 24; -const WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS: u32 = 24; const DIRECTION_WARNING_MARGIN_X100: u32 = 90; const RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100: u32 = 320; const INFORMATIVE_SPAN_ROW_SAMPLES: u32 = 24; @@ -254,10 +253,7 @@ pub(crate) struct ScrollSession { preview_width_px: u32, } impl ScrollSession { - pub(crate) fn new( - base_frame: RgbaImage, - preview_width_px: u32, - ) -> color_eyre::eyre::Result { + pub(crate) fn new(base_frame: RgbaImage, preview_width_px: u32) -> Result { let fingerprint = scroll_capture_fingerprint(&base_frame); let anchor_preview = self::support::resize_strip_to_preview_width(&base_frame, preview_width_px.max(1)); @@ -326,7 +322,7 @@ impl ScrollSession { pub(crate) fn observe_downward_sample( &mut self, frame: RgbaImage, - ) -> color_eyre::eyre::Result { + ) -> Result { self.observe_downward_sample_with_motion_hint(frame, None) } @@ -334,7 +330,7 @@ impl ScrollSession { &mut self, frame: RgbaImage, motion_rows_hint: Option, - ) -> color_eyre::eyre::Result { + ) -> Result { self.observe_downward_sample_with_motion_hint_and_burst(frame, motion_rows_hint, false) } @@ -343,7 +339,7 @@ impl ScrollSession { frame: RgbaImage, motion_rows_hint: Option, allow_burst_search: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { self.observe_sample_with_motion_context( frame, ScrollDirection::Down, @@ -356,447 +352,17 @@ impl ScrollSession { pub(crate) fn observe_upward_sample( &mut self, frame: RgbaImage, - ) -> color_eyre::eyre::Result { + ) -> Result { self.observe_sample_with_motion_context(frame, ScrollDirection::Up, None, false) } - pub(crate) fn observe_worker_pairwise_vision_frame( - &mut self, - frame: RgbaImage, - ) -> color_eyre::eyre::Result { - self.clear_last_downward_sample_registration(); - - if frame.width() != self.anchor_frame.width() { - return Err(eyre::eyre!( - "frame width mismatch: expected {} got {}", - self.anchor_frame.width(), - frame.width() - )); - } - - let fingerprint = scroll_capture_fingerprint(&frame); - let previous_worker_frame = self.worker_pairwise_previous_frame.clone(); - - if self.worker_pairwise_requires_committed_reacquire { - if frame == self.last_committed_frame { - self.worker_pairwise_requires_committed_reacquire = false; - - return Ok(self.observe_worker_pairwise_no_change( - frame, - fingerprint, - "worker_pairwise_reacquired_last_committed_frame", - )); - } - - if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { - self.worker_pairwise_requires_committed_reacquire = false; - - return self.observe_resolved_worker_pairwise_downward_motion( - frame, - fingerprint, - motion_rows, - Some(motion_rows), - ); - } - - return Ok(self.block_worker_pairwise_until_committed_reacquire(frame, fingerprint)); - } - if frame == previous_worker_frame { - let reason = if frame == self.last_committed_frame { - "frame_matches_last_committed_frame" - } else { - "frame_matches_worker_pairwise_previous_frame" - }; - - return Ok(self.observe_worker_pairwise_no_change(frame, fingerprint, reason)); - } - - let vision_match = self::support::classify_vision_downward_sample_motion_against( - &previous_worker_frame, - &frame, - ); - let (matched, corroborated_shift_rows) = if let Some(matched) = vision_match { - ( - matched, - self::support::trusted_pairwise_downward_shift_rows_near_motion( - &previous_worker_frame, - &frame, - matched.motion_rows, - WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, - ), - ) - } else if let Some(matched) = - self::support::trusted_pairwise_downward_shift_match(&previous_worker_frame, &frame) - { - let max_pixel_fallback_motion_rows = - previous_worker_frame.height().saturating_div(2).max(1); - - if matched.motion_rows > max_pixel_fallback_motion_rows { - let candidate_viewport_top_y = self - .current_viewport_top_y - .saturating_add(i32::try_from(matched.motion_rows).unwrap_or_default()); - let growth_rows = - self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); - - return Ok(self.block_worker_pairwise_growth( - frame, - fingerprint, - matched.motion_rows, - candidate_viewport_top_y, - growth_rows, - "worker_pairwise_pixel_overlap_exceeded_fallback_budget", - )); - } - - (matched, Some(matched.motion_rows)) - } else { - if let Some(upward_motion_rows) = - self::support::trusted_pairwise_upward_shift_rows(&previous_worker_frame, &frame) - { - return Ok(self.observe_worker_pairwise_upward_motion( - frame, - fingerprint, - upward_motion_rows, - )); - } - if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { - return self.observe_resolved_worker_pairwise_downward_motion( - frame, - fingerprint, - motion_rows, - Some(motion_rows), - ); - } - - return Ok(self.observe_worker_pairwise_no_change( - frame, - fingerprint, - "worker_pairwise_no_downward_offset", - )); - }; - - self.observe_resolved_worker_pairwise_downward_motion( - frame, - fingerprint, - matched.motion_rows, - corroborated_shift_rows, - ) - } - - pub(crate) fn observe_worker_pairwise_vision_frame_with_motion_hint( - &mut self, - frame: RgbaImage, - motion_rows_hint: Option, - ) -> color_eyre::eyre::Result { - let previous_hint = self.transient_motion_rows_hint; - - self.transient_motion_rows_hint = motion_rows_hint; - - self.record_last_sample_eval_context(); - - let result = self.observe_worker_pairwise_vision_frame(frame); - - self.transient_motion_rows_hint = previous_hint; - - result - } - - fn worker_pairwise_committed_catchup_motion_rows(&self, frame: &RgbaImage) -> Option { - if frame == &self.last_committed_frame { - return None; - } - - let hinted_match = self.normalized_transient_motion_rows_hint().and_then(|hint| { - let tolerance = (hint / 2).clamp( - WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, - INITIAL_DOWNWARD_MAX_MOTION_ROWS, - ); - - self::support::trusted_pairwise_downward_shift_rows_near_motion( - &self.last_committed_frame, - frame, - hint, - tolerance, - ) - }); - let fallback_match = || { - self::support::trusted_pairwise_downward_shift_match(&self.last_committed_frame, frame) - .map(|matched| matched.motion_rows) - }; - - hinted_match - .or_else(fallback_match) - .filter(|motion_rows| *motion_rows >= WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS) - } - - fn observe_resolved_worker_pairwise_downward_motion( - &mut self, - frame: RgbaImage, - fingerprint: Vec, - vision_motion_rows: u32, - corroborated_shift_rows: Option, - ) -> color_eyre::eyre::Result { - let effective_motion_rows = match Self::resolve_worker_pairwise_motion_rows( - vision_motion_rows, - corroborated_shift_rows, - ) { - Ok(motion_rows) => motion_rows, - Err(block_reason) => { - return Ok(self.block_worker_pairwise_growth( - frame, - fingerprint, - vision_motion_rows, - self.current_viewport_top_y, - 0, - block_reason, - )); - }, - }; - - tracing::debug!( - op = "scroll_capture.worker_pairwise_motion_resolved", - vision_motion_rows, - corroborated_motion_rows = corroborated_shift_rows, - effective_motion_rows, - current_viewport_top_y = self.current_viewport_top_y, - observed_viewport_top_y = self.observed_viewport_top_y, - "Scroll-capture worker pairwise motion resolved against pixel overlap." - ); - - if effective_motion_rows == 0 { - return Ok(self.block_worker_pairwise_growth( - frame, - fingerprint, - effective_motion_rows, - self.current_viewport_top_y, - 0, - "worker_pairwise_zero_effective_motion", - )); - } - - let candidate_viewport_top_y = self - .current_viewport_top_y - .saturating_add(i32::try_from(effective_motion_rows).unwrap_or_default()); - let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); - let frame_max_growth_rows = frame.height().saturating_sub(1).max(1); - - if growth_rows == 0 || growth_rows > frame_max_growth_rows { - return Ok(self.block_worker_pairwise_growth( - frame, - fingerprint, - effective_motion_rows, - candidate_viewport_top_y, - growth_rows, - "worker_pairwise_growth_exceeded_frame_bounds", - )); - } - - self.log_decision( - "scroll_capture.worker_pairwise_growth_candidate", - ScrollDirection::Down, - Some(MotionObservation { - direction: ScrollDirection::Down, - motion_rows: effective_motion_rows, - }), - Some(candidate_viewport_top_y), - Some(growth_rows), - Some("worker_pairwise_vision"), - ); - - self.worker_pairwise_previous_frame = frame.clone(); - self.worker_pairwise_requires_committed_reacquire = false; - - self.clear_preview_only_downward_recovery_carryover(); - - if self.resume_frontier_top_y.is_some() { - let outcome = self.observe_downward_motion_while_resume_frontier_active( - frame.clone(), - effective_motion_rows, - true, - )?; - - if !matches!( - outcome, - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, .. } - ) { - self.record_last_sample(&frame, fingerprint); - } - - return Ok(outcome); - } - - self.apply_growth( - frame.clone(), - growth_rows, - candidate_viewport_top_y, - "worker_pairwise_vision", - Some(vision_motion_rows), - Some(effective_motion_rows), - None, - ) - } - - fn observe_worker_pairwise_no_change( - &mut self, - frame: RgbaImage, - fingerprint: Vec, - reason: &'static str, - ) -> ScrollObserveOutcome { - if reason == "worker_pairwise_no_downward_offset" && frame != self.last_committed_frame { - self.record_last_sample(&frame, fingerprint); - self.clear_preview_only_downward_recovery_carryover(); - - self.worker_pairwise_requires_committed_reacquire = true; - - self.log_decision( - "scroll_capture.worker_pairwise_no_change", - ScrollDirection::Down, - None, - Some(self.observed_viewport_top_y), - Some(0), - Some(reason), - ); - - return ScrollObserveOutcome::NoChange; - } - - self.update_worker_pairwise_reference_frame(frame, fingerprint); - self.log_decision( - "scroll_capture.worker_pairwise_no_change", - ScrollDirection::Down, - None, - Some(self.observed_viewport_top_y), - Some(0), - Some(reason), - ); - - ScrollObserveOutcome::NoChange - } - - fn observe_worker_pairwise_upward_motion( - &mut self, - frame: RgbaImage, - fingerprint: Vec, - motion_rows: u32, - ) -> ScrollObserveOutcome { - self.record_last_sample(&frame, fingerprint); - self.clear_preview_only_downward_recovery_carryover(); - - if self.current_viewport_top_y <= 0 && self.resume_frontier_top_y.is_none() { - if frame != self.last_committed_frame { - self.worker_pairwise_requires_committed_reacquire = true; - } - - self.log_decision( - "scroll_capture.worker_pairwise_upward_at_top", - ScrollDirection::Up, - Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), - Some(self.current_viewport_top_y), - Some(0), - Some("worker_pairwise_upward_motion_without_committed_growth"), - ); - - return ScrollObserveOutcome::PreviewUpdated; - } - - self.worker_pairwise_previous_frame = frame.clone(); - - self.observe_upward_rewind(motion_rows); - self.log_decision( - "scroll_capture.worker_pairwise_rewind_armed", - ScrollDirection::Up, - Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), - None, - None, - Some("worker_pairwise_detected_upward_motion"), - ); - - ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } - } - - fn block_worker_pairwise_growth( - &mut self, - frame: RgbaImage, - fingerprint: Vec, - motion_rows: u32, - candidate_viewport_top_y: i32, - growth_rows: u32, - reason: &'static str, - ) -> ScrollObserveOutcome { - self.record_last_sample(&frame, fingerprint); - self.clear_preview_only_downward_recovery_carryover(); - - if motion_rows > 0 { - self.worker_pairwise_requires_committed_reacquire = true; - } - - self.log_decision( - "scroll_capture.worker_pairwise_growth_blocked", - ScrollDirection::Down, - Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), - Some(candidate_viewport_top_y), - Some(growth_rows), - Some(reason), - ); - - ScrollObserveOutcome::NoChange - } - - fn block_worker_pairwise_until_committed_reacquire( - &mut self, - frame: RgbaImage, - fingerprint: Vec, - ) -> ScrollObserveOutcome { - self.record_last_sample(&frame, fingerprint); - self.clear_preview_only_downward_recovery_carryover(); - self.log_decision( - "scroll_capture.worker_pairwise_growth_blocked", - ScrollDirection::Down, - None, - Some(self.current_viewport_top_y), - Some(0), - Some("worker_pairwise_requires_committed_reacquire_after_blocked_gap"), - ); - - ScrollObserveOutcome::NoChange - } - - fn resolve_worker_pairwise_motion_rows( - vision_motion_rows: u32, - corroborated_shift_rows: Option, - ) -> std::result::Result { - let Some(corroborated_shift_rows) = corroborated_shift_rows else { - return Err("worker_pairwise_missing_or_ambiguous_overlap_corroboration"); - }; - - if corroborated_shift_rows == 0 { - return Err("worker_pairwise_zero_overlap_corroboration"); - } - if vision_motion_rows.abs_diff(corroborated_shift_rows) - > WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS - { - return Err("worker_pairwise_vision_overlap_motion_mismatch"); - } - - Ok(corroborated_shift_rows) - } - - fn update_worker_pairwise_reference_frame(&mut self, frame: RgbaImage, fingerprint: Vec) { - self.record_last_sample(&frame, fingerprint.clone()); - self.record_last_downward_observed_sample(&frame, fingerprint); - - self.worker_pairwise_previous_frame = frame; - - self.clear_preview_only_downward_recovery_carryover(); - } - fn observe_sample_with_motion_context( &mut self, frame: RgbaImage, input_direction: ScrollDirection, motion_rows_hint: Option, allow_burst_search: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { let previous_hint = self.transient_motion_rows_hint; let previous_burst = self.transient_burst_search_enabled; @@ -817,7 +383,7 @@ impl ScrollSession { &mut self, frame: RgbaImage, input_direction: ScrollDirection, - ) -> color_eyre::eyre::Result { + ) -> Result { self.clear_last_downward_sample_registration(); if frame.width() != self.anchor_frame.width() { @@ -1029,7 +595,7 @@ impl ScrollSession { sample_motion: Option, downward_sample_match: Option, preview_changed: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { self.last_unconfirmed_upward_fingerprint = None; if let Some(motion) = sample_motion { @@ -1102,7 +668,7 @@ impl ScrollSession { sample_delta: Option, sample_motion: Option, _preview_changed: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { let diagnostics = self.diagnose_upward_input(&frame); self.log_upward_input_diagnostics(&diagnostics, sample_delta, sample_motion, &frame); @@ -1720,7 +1286,7 @@ impl ScrollSession { frame: RgbaImage, observed_match: DownwardSampleMatch, preview_changed: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { let motion_rows = observed_match.matched.motion_rows; if self.resume_frontier_top_y.is_some() { @@ -1791,7 +1357,7 @@ impl ScrollSession { motion_rows: u32, candidate: DownwardViewportCandidate, preview_changed: bool, - ) -> color_eyre::eyre::Result> { + ) -> Result> { if self.transient_burst_candidate_underconsumes_input_hint(candidate) { return Ok(Some(self.block_downward_growth_candidate( frame, @@ -1872,7 +1438,7 @@ impl ScrollSession { observed_match: DownwardSampleMatch, motion_rows: u32, preview_changed: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { let reset_preview_only_local_baseline = self.should_reset_preview_only_local_baseline_after_huge_far_committed_block(); let preview_only_local_viewport_top_y = if self @@ -1947,7 +1513,7 @@ impl ScrollSession { candidate: DownwardViewportCandidate, preview_changed: bool, block_reason: &'static str, - ) -> color_eyre::eyre::Result { + ) -> Result { self.consecutive_transient_burst_missing_downward_candidate_frames = 0; self.refresh_local_downward_sample(frame); @@ -2114,7 +1680,7 @@ impl ScrollSession { frame: RgbaImage, motion_rows: u32, preview_changed: bool, - ) -> color_eyre::eyre::Result { + ) -> Result { let candidate_observed_viewport_top_y = self .observed_viewport_top_y .saturating_add(i32::try_from(motion_rows).unwrap_or_default()); @@ -2287,7 +1853,7 @@ impl ScrollSession { preview_changed: bool, frame_reacquires_last_committed_viewport: bool, context: ResumeFrontierDirectMatchContext, - ) -> color_eyre::eyre::Result { + ) -> Result { let direct_match_hint_rows = Some(self.resume_frontier_direct_match_hint_rows(context)); let raw_committed_down_match = self.evaluate_reference_overlap_direction_preferred_only( &self.last_committed_frame, @@ -2475,7 +2041,7 @@ impl ScrollSession { preview_changed: bool, down: DirectionMatch, context: ResumeFrontierDirectMatchContext, - ) -> color_eyre::eyre::Result { + ) -> Result { let candidate_viewport_top_y = if self.resume_frontier_requires_reacquire { let resume_frontier_top_y = self.resume_frontier_top_y.unwrap_or(self.current_viewport_top_y); @@ -2528,7 +2094,7 @@ impl ScrollSession { preview_changed: bool, detected_motion: Option, decision_source: &'static str, - ) -> color_eyre::eyre::Result { + ) -> Result { let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); let effective_motion_rows_hint = self.effective_motion_rows_hint(); diff --git a/packages/rsnap-overlay/src/scroll_capture/support.rs b/packages/rsnap-overlay/src/scroll_capture/support.rs index 63a5cb07..b38417d3 100644 --- a/packages/rsnap-overlay/src/scroll_capture/support.rs +++ b/packages/rsnap-overlay/src/scroll_capture/support.rs @@ -48,6 +48,25 @@ const MOTION_OVERLAP_MIN_MATCHING_COLUMN_PERCENT: u32 = 80; const MOTION_OVERLAP_BAD_EDGE_SAMPLE_DIVISOR: usize = 10; const MOTION_OVERLAP_BAD_EDGE_MIN_SAMPLES: usize = 8; +#[derive(Clone, Copy, Debug)] +struct MotionCoverageColumnScore { + structure_score: u32, + motion_score: u32, +} +impl MotionCoverageColumnScore { + fn has_structure(self, threshold: u32) -> bool { + self.structure_score >= threshold + } + + fn has_motion(self, threshold: u32) -> bool { + self.motion_score >= threshold + } + + fn is_static(self, structure_threshold: u32, motion_threshold: u32) -> bool { + self.has_structure(structure_threshold) && self.motion_score <= motion_threshold + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum OverlapOrientation { PreviousBottomToNextTop, @@ -1179,7 +1198,7 @@ fn motion_coverage_supports_structural_span( max_structure_score = max_structure_score.max(structure_score); max_motion_score = max_motion_score.max(motion_score); - scores.push((structure_score, motion_score)); + scores.push(MotionCoverageColumnScore { structure_score, motion_score }); } if max_structure_score == 0 || max_motion_score == 0 { @@ -1205,14 +1224,14 @@ fn motion_coverage_supports_structural_span( return false; } - for &(structure_score, motion_score) in span_scores { - if structure_score < structure_threshold { + for &score in span_scores { + if !score.has_structure(structure_threshold) { continue; } informative_columns = informative_columns.saturating_add(1); - if motion_score >= motion_threshold { + if score.has_motion(motion_threshold) { moving_informative_columns = moving_informative_columns.saturating_add(1); } } @@ -1223,7 +1242,7 @@ fn motion_coverage_supports_structural_span( } fn raw_frame_pair_has_static_informative_band( - scores: &[(u32, u32)], + scores: &[MotionCoverageColumnScore], max_structure_score: u32, max_motion_score: u32, ) -> bool { @@ -1242,15 +1261,14 @@ fn raw_frame_pair_has_static_informative_band( let mut moving_end = None; let mut static_flags = Vec::with_capacity(scores.len()); - for (column, (structure_score, motion_score)) in scores.iter().enumerate() { - if *structure_score >= structure_threshold && *motion_score >= moving_motion_threshold { + for (column, score) in scores.iter().enumerate() { + if score.has_structure(structure_threshold) && score.has_motion(moving_motion_threshold) { moving_start.get_or_insert(column); moving_end = Some(column.saturating_add(1)); } - static_flags - .push(*structure_score >= structure_threshold && *motion_score <= motion_threshold); + static_flags.push(score.is_static(structure_threshold, motion_threshold)); } let Some(moving_start) = moving_start else { @@ -1302,7 +1320,7 @@ fn static_side_band_has_enough_columns( } fn raw_frame_pair_has_static_informative_edge( - scores: &[(u32, u32)], + scores: &[MotionCoverageColumnScore], structure_threshold: u32, motion_threshold: u32, left_leading_columns: u32, @@ -1329,14 +1347,14 @@ fn raw_static_edge_run_len( leading_columns: u32, ) -> u32 where - I: IntoIterator, + I: IntoIterator, { let mut skipped_columns = leading_columns; let mut static_columns = 0_u32; let mut seen_informative = false; - for (structure_score, motion_score) in iter { - if structure_score < structure_threshold { + for score in iter { + if !score.has_structure(structure_threshold) { if seen_informative { break; } @@ -1355,7 +1373,7 @@ where seen_informative = true; - if motion_score >= motion_threshold { + if score.has_motion(motion_threshold) { break; } diff --git a/packages/rsnap-overlay/src/scroll_capture/tests.rs b/packages/rsnap-overlay/src/scroll_capture/tests.rs index 980152d5..5fa8803e 100644 --- a/packages/rsnap-overlay/src/scroll_capture/tests.rs +++ b/packages/rsnap-overlay/src/scroll_capture/tests.rs @@ -278,6 +278,36 @@ fn assert_worker_pairwise_blocked_overshot_does_not_commit_tail( assert_eq!(session.current_viewport_top_y(), 0); } +fn assert_pairwise_rejects_static_selection(previous: &image::RgbaImage, next: &image::RgbaImage) { + assert_eq!(support::estimate_pairwise_downward_shift_rows(previous, next), None); +} + +#[cfg(target_os = "macos")] +fn assert_worker_pairwise_rejects_static_selection( + base: image::RgbaImage, + next: image::RgbaImage, + rejected_growth_rows: u32, +) { + let mut session = match ScrollSession::new(base.clone(), 320) { + Ok(session) => session, + Err(err) => panic!("static-selection fixture should create session: {err:#}"), + }; + let outcome = match session.observe_worker_pairwise_vision_frame(next) { + Ok(outcome) => outcome, + Err(err) => panic!("static-selection fixture should observe frame: {err:#}"), + }; + + assert_ne!( + outcome, + ScrollObserveOutcome::Committed { + direction: ScrollDirection::Down, + growth_rows: rejected_growth_rows, + } + ); + assert_eq!(session.current_viewport_top_y(), 0); + assert_eq!(session.export_image(), &base); +} + #[test] fn overlap_detection_prefers_largest_matching_suffix() { let previous = make_test_image( @@ -612,72 +642,37 @@ fn dynamic_scroll_center_does_not_stitch_when_static_sidebars_are_in_selection() } #[test] -fn pairwise_shift_estimate_rejects_narrow_dynamic_center_with_static_sidebars_present() { - let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); - let moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); - - assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &moved), None); -} - -#[test] -fn pairwise_shift_estimate_rejects_wide_dynamic_center_with_static_right_rail_present() { - let base = make_codex_like_right_static_rail_frame(640, 360, 0); - let moved = make_codex_like_right_static_rail_frame(640, 360, 72); - - assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &moved), None); -} - -#[test] -fn pairwise_shift_estimate_ignores_static_sidebars_without_center_scroll_match() { - let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); +fn pairwise_shift_estimate_rejects_static_selection_cases() { + let narrow_base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); + let narrow_moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); + let codex_base = make_codex_like_right_static_rail_frame(640, 360, 0); + let codex_moved = make_codex_like_right_static_rail_frame(640, 360, 72); + let unrelated_base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); let unrelated_center = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 2, false); - assert_eq!(support::estimate_pairwise_downward_shift_rows(&base, &unrelated_center), None); + assert_pairwise_rejects_static_selection(&narrow_base, &narrow_moved); + assert_pairwise_rejects_static_selection(&codex_base, &codex_moved); + assert_pairwise_rejects_static_selection(&unrelated_base, &unrelated_center); } #[cfg(target_os = "macos")] #[test] -fn worker_pairwise_vision_rejects_narrow_dynamic_center_with_static_sidebars_present() { - let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true); - let moved = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true); - let mut session = ScrollSession::new(base.clone(), 320).unwrap(); - - assert_ne!( - session.observe_worker_pairwise_vision_frame(moved).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } +fn worker_pairwise_vision_rejects_static_selection_cases() { + assert_worker_pairwise_rejects_static_selection( + make_static_sidebar_center_frame(320, 240, 145, 30, 0, 7, true), + make_static_sidebar_center_frame(320, 240, 145, 30, 72, 7, true), + 72, ); - assert_eq!(session.current_viewport_top_y(), 0); - assert_eq!(session.export_image(), &base); -} - -#[cfg(target_os = "macos")] -#[test] -fn worker_pairwise_vision_rejects_wide_dynamic_center_with_static_right_rail_present() { - let base = make_codex_like_right_static_rail_frame(640, 360, 0); - let moved = make_codex_like_right_static_rail_frame(640, 360, 72); - let mut session = ScrollSession::new(base.clone(), 320).unwrap(); - - assert_ne!( - session.observe_worker_pairwise_vision_frame(moved).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } + assert_worker_pairwise_rejects_static_selection( + make_codex_like_right_static_rail_frame(640, 360, 0), + make_codex_like_right_static_rail_frame(640, 360, 72), + 72, ); - assert_eq!(session.current_viewport_top_y(), 0); - assert_eq!(session.export_image(), &base); -} - -#[cfg(target_os = "macos")] -#[test] -fn worker_pairwise_vision_ignores_static_sidebars_without_center_scroll_match() { - let base = make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false); - let unrelated_center = make_static_sidebar_center_frame(320, 240, 145, 30, 72, 2, false); - let mut session = ScrollSession::new(base.clone(), 320).unwrap(); - - assert_ne!( - session.observe_worker_pairwise_vision_frame(unrelated_center).unwrap(), - ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, growth_rows: 72 } + assert_worker_pairwise_rejects_static_selection( + make_static_sidebar_center_frame(320, 240, 145, 30, 0, 1, false), + make_static_sidebar_center_frame(320, 240, 145, 30, 72, 2, false), + 72, ); - assert_eq!(session.current_viewport_top_y(), 0); - assert_eq!(session.export_image(), &base); } #[test] diff --git a/packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs b/packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs new file mode 100644 index 00000000..edb694e9 --- /dev/null +++ b/packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs @@ -0,0 +1,440 @@ +use color_eyre::eyre; +use image::RgbaImage; + +use crate::scroll_capture::{ + self, INITIAL_DOWNWARD_MAX_MOTION_ROWS, MotionObservation, ScrollDirection, + ScrollObserveOutcome, ScrollSession, support, +}; + +const WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS: u32 = 24; +const WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS: u32 = 24; + +impl ScrollSession { + pub(crate) fn observe_worker_pairwise_vision_frame( + &mut self, + frame: RgbaImage, + ) -> color_eyre::eyre::Result { + self.clear_last_downward_sample_registration(); + + if frame.width() != self.anchor_frame.width() { + return Err(eyre::eyre!( + "frame width mismatch: expected {} got {}", + self.anchor_frame.width(), + frame.width() + )); + } + + let fingerprint = scroll_capture::scroll_capture_fingerprint(&frame); + let previous_worker_frame = self.worker_pairwise_previous_frame.clone(); + + if self.worker_pairwise_requires_committed_reacquire { + if frame == self.last_committed_frame { + self.worker_pairwise_requires_committed_reacquire = false; + + return Ok(self.observe_worker_pairwise_no_change( + frame, + fingerprint, + "worker_pairwise_reacquired_last_committed_frame", + )); + } + + if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { + self.worker_pairwise_requires_committed_reacquire = false; + + return self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, + motion_rows, + Some(motion_rows), + ); + } + + return Ok(self.block_worker_pairwise_until_committed_reacquire(frame, fingerprint)); + } + if frame == previous_worker_frame { + let reason = if frame == self.last_committed_frame { + "frame_matches_last_committed_frame" + } else { + "frame_matches_worker_pairwise_previous_frame" + }; + + return Ok(self.observe_worker_pairwise_no_change(frame, fingerprint, reason)); + } + + let vision_match = + support::classify_vision_downward_sample_motion_against(&previous_worker_frame, &frame); + let (matched, corroborated_shift_rows) = if let Some(matched) = vision_match { + ( + matched, + support::trusted_pairwise_downward_shift_rows_near_motion( + &previous_worker_frame, + &frame, + matched.motion_rows, + WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, + ), + ) + } else if let Some(matched) = + support::trusted_pairwise_downward_shift_match(&previous_worker_frame, &frame) + { + let max_pixel_fallback_motion_rows = + previous_worker_frame.height().saturating_div(2).max(1); + + if matched.motion_rows > max_pixel_fallback_motion_rows { + let candidate_viewport_top_y = self + .current_viewport_top_y + .saturating_add(i32::try_from(matched.motion_rows).unwrap_or_default()); + let growth_rows = + self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + + return Ok(self.block_worker_pairwise_growth( + frame, + fingerprint, + matched.motion_rows, + candidate_viewport_top_y, + growth_rows, + "worker_pairwise_pixel_overlap_exceeded_fallback_budget", + )); + } + + (matched, Some(matched.motion_rows)) + } else { + if let Some(upward_motion_rows) = + support::trusted_pairwise_upward_shift_rows(&previous_worker_frame, &frame) + { + return Ok(self.observe_worker_pairwise_upward_motion( + frame, + fingerprint, + upward_motion_rows, + )); + } + if let Some(motion_rows) = self.worker_pairwise_committed_catchup_motion_rows(&frame) { + return self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, + motion_rows, + Some(motion_rows), + ); + } + + return Ok(self.observe_worker_pairwise_no_change( + frame, + fingerprint, + "worker_pairwise_no_downward_offset", + )); + }; + + self.observe_resolved_worker_pairwise_downward_motion( + frame, + fingerprint, + matched.motion_rows, + corroborated_shift_rows, + ) + } + + pub(crate) fn observe_worker_pairwise_vision_frame_with_motion_hint( + &mut self, + frame: RgbaImage, + motion_rows_hint: Option, + ) -> color_eyre::eyre::Result { + let previous_hint = self.transient_motion_rows_hint; + + self.transient_motion_rows_hint = motion_rows_hint; + + self.record_last_sample_eval_context(); + + let result = self.observe_worker_pairwise_vision_frame(frame); + + self.transient_motion_rows_hint = previous_hint; + + result + } + + fn worker_pairwise_committed_catchup_motion_rows(&self, frame: &RgbaImage) -> Option { + if frame == &self.last_committed_frame { + return None; + } + + let hinted_match = self.normalized_transient_motion_rows_hint().and_then(|hint| { + let tolerance = (hint / 2).clamp( + WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS, + INITIAL_DOWNWARD_MAX_MOTION_ROWS, + ); + + support::trusted_pairwise_downward_shift_rows_near_motion( + &self.last_committed_frame, + frame, + hint, + tolerance, + ) + }); + let fallback_match = || { + support::trusted_pairwise_downward_shift_match(&self.last_committed_frame, frame) + .map(|matched| matched.motion_rows) + }; + + hinted_match + .or_else(fallback_match) + .filter(|motion_rows| *motion_rows >= WORKER_PAIRWISE_COMMITTED_CATCHUP_MIN_MOTION_ROWS) + } + + fn observe_resolved_worker_pairwise_downward_motion( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + vision_motion_rows: u32, + corroborated_shift_rows: Option, + ) -> color_eyre::eyre::Result { + let effective_motion_rows = match Self::resolve_worker_pairwise_motion_rows( + vision_motion_rows, + corroborated_shift_rows, + ) { + Ok(motion_rows) => motion_rows, + Err(block_reason) => { + return Ok(self.block_worker_pairwise_growth( + frame, + fingerprint, + vision_motion_rows, + self.current_viewport_top_y, + 0, + block_reason, + )); + }, + }; + + tracing::debug!( + op = "scroll_capture.worker_pairwise_motion_resolved", + vision_motion_rows, + corroborated_motion_rows = corroborated_shift_rows, + effective_motion_rows, + current_viewport_top_y = self.current_viewport_top_y, + observed_viewport_top_y = self.observed_viewport_top_y, + "Scroll-capture worker pairwise motion resolved against pixel overlap." + ); + + if effective_motion_rows == 0 { + return Ok(self.block_worker_pairwise_growth( + frame, + fingerprint, + effective_motion_rows, + self.current_viewport_top_y, + 0, + "worker_pairwise_zero_effective_motion", + )); + } + + let candidate_viewport_top_y = self + .current_viewport_top_y + .saturating_add(i32::try_from(effective_motion_rows).unwrap_or_default()); + let growth_rows = self.growth_rows_for_candidate_viewport_top_y(candidate_viewport_top_y); + let frame_max_growth_rows = frame.height().saturating_sub(1).max(1); + + if growth_rows == 0 || growth_rows > frame_max_growth_rows { + return Ok(self.block_worker_pairwise_growth( + frame, + fingerprint, + effective_motion_rows, + candidate_viewport_top_y, + growth_rows, + "worker_pairwise_growth_exceeded_frame_bounds", + )); + } + + self.log_decision( + "scroll_capture.worker_pairwise_growth_candidate", + ScrollDirection::Down, + Some(MotionObservation { + direction: ScrollDirection::Down, + motion_rows: effective_motion_rows, + }), + Some(candidate_viewport_top_y), + Some(growth_rows), + Some("worker_pairwise_vision"), + ); + + self.worker_pairwise_previous_frame = frame.clone(); + self.worker_pairwise_requires_committed_reacquire = false; + + self.clear_preview_only_downward_recovery_carryover(); + + if self.resume_frontier_top_y.is_some() { + let outcome = self.observe_downward_motion_while_resume_frontier_active( + frame.clone(), + effective_motion_rows, + true, + )?; + + if !matches!( + outcome, + ScrollObserveOutcome::Committed { direction: ScrollDirection::Down, .. } + ) { + self.record_last_sample(&frame, fingerprint); + } + + return Ok(outcome); + } + + self.apply_growth( + frame.clone(), + growth_rows, + candidate_viewport_top_y, + "worker_pairwise_vision", + Some(vision_motion_rows), + Some(effective_motion_rows), + None, + ) + } + + fn observe_worker_pairwise_no_change( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + reason: &'static str, + ) -> ScrollObserveOutcome { + if reason == "worker_pairwise_no_downward_offset" && frame != self.last_committed_frame { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + self.worker_pairwise_requires_committed_reacquire = true; + + self.log_decision( + "scroll_capture.worker_pairwise_no_change", + ScrollDirection::Down, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some(reason), + ); + + return ScrollObserveOutcome::NoChange; + } + + self.update_worker_pairwise_reference_frame(frame, fingerprint); + self.log_decision( + "scroll_capture.worker_pairwise_no_change", + ScrollDirection::Down, + None, + Some(self.observed_viewport_top_y), + Some(0), + Some(reason), + ); + + ScrollObserveOutcome::NoChange + } + + fn observe_worker_pairwise_upward_motion( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + motion_rows: u32, + ) -> ScrollObserveOutcome { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + if self.current_viewport_top_y <= 0 && self.resume_frontier_top_y.is_none() { + if frame != self.last_committed_frame { + self.worker_pairwise_requires_committed_reacquire = true; + } + + self.log_decision( + "scroll_capture.worker_pairwise_upward_at_top", + ScrollDirection::Up, + Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), + Some(self.current_viewport_top_y), + Some(0), + Some("worker_pairwise_upward_motion_without_committed_growth"), + ); + + return ScrollObserveOutcome::PreviewUpdated; + } + + self.worker_pairwise_previous_frame = frame.clone(); + + self.observe_upward_rewind(motion_rows); + self.log_decision( + "scroll_capture.worker_pairwise_rewind_armed", + ScrollDirection::Up, + Some(MotionObservation { direction: ScrollDirection::Up, motion_rows }), + None, + None, + Some("worker_pairwise_detected_upward_motion"), + ); + + ScrollObserveOutcome::UnsupportedDirection { direction: ScrollDirection::Up } + } + + fn block_worker_pairwise_growth( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + motion_rows: u32, + candidate_viewport_top_y: i32, + growth_rows: u32, + reason: &'static str, + ) -> ScrollObserveOutcome { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + + if motion_rows > 0 { + self.worker_pairwise_requires_committed_reacquire = true; + } + + self.log_decision( + "scroll_capture.worker_pairwise_growth_blocked", + ScrollDirection::Down, + Some(MotionObservation { direction: ScrollDirection::Down, motion_rows }), + Some(candidate_viewport_top_y), + Some(growth_rows), + Some(reason), + ); + + ScrollObserveOutcome::NoChange + } + + fn block_worker_pairwise_until_committed_reacquire( + &mut self, + frame: RgbaImage, + fingerprint: Vec, + ) -> ScrollObserveOutcome { + self.record_last_sample(&frame, fingerprint); + self.clear_preview_only_downward_recovery_carryover(); + self.log_decision( + "scroll_capture.worker_pairwise_growth_blocked", + ScrollDirection::Down, + None, + Some(self.current_viewport_top_y), + Some(0), + Some("worker_pairwise_requires_committed_reacquire_after_blocked_gap"), + ); + + ScrollObserveOutcome::NoChange + } + + pub(crate) fn resolve_worker_pairwise_motion_rows( + vision_motion_rows: u32, + corroborated_shift_rows: Option, + ) -> std::result::Result { + let Some(corroborated_shift_rows) = corroborated_shift_rows else { + return Err("worker_pairwise_missing_or_ambiguous_overlap_corroboration"); + }; + + if corroborated_shift_rows == 0 { + return Err("worker_pairwise_zero_overlap_corroboration"); + } + if vision_motion_rows.abs_diff(corroborated_shift_rows) + > WORKER_PAIRWISE_CORROBORATION_TOLERANCE_ROWS + { + return Err("worker_pairwise_vision_overlap_motion_mismatch"); + } + + Ok(corroborated_shift_rows) + } + + fn update_worker_pairwise_reference_frame(&mut self, frame: RgbaImage, fingerprint: Vec) { + self.record_last_sample(&frame, fingerprint.clone()); + self.record_last_downward_observed_sample(&frame, fingerprint); + + self.worker_pairwise_previous_frame = frame; + + self.clear_preview_only_downward_recovery_carryover(); + } +} diff --git a/scripts/smoke/native-scroll-capture-macos.sh b/scripts/smoke/native-scroll-capture-macos.sh index 48ff0a83..51ee2f00 100755 --- a/scripts/smoke/native-scroll-capture-macos.sh +++ b/scripts/smoke/native-scroll-capture-macos.sh @@ -55,6 +55,7 @@ live_hud_assert_interactive_session live_hud_assert_shareable_display ROOT_DIR="$(live_hud_repo_root)" live_hud_init_environment "$ROOT_DIR" +COMMON_ROOT="$(cd "$(git -C "$ROOT_DIR" rev-parse --git-common-dir)/.." && pwd)" smoke_log() { printf '[smoke] +%ss %s\n' "$SECONDS" "$*" @@ -65,6 +66,7 @@ PREF_SNAPSHOT="$(mktemp "${TMPDIR:-/tmp}/rsnap-scroll-prefs.XXXXXX.plist")" PREF_SNAPSHOT_EXISTS=0 SCROLL_BACKGROUND_PID="" SCROLL_BACKGROUND_READY="$(mktemp "${TMPDIR:-/tmp}/rsnap-scroll-bg.XXXXXX.ready")" +RSNAP_SMOKE_APP_EXECUTABLE="${RSNAP_NATIVE_HOST_STAGE_DIR:-$COMMON_ROOT/target/rsnap-native-host}/Rsnap.app/Contents/MacOS/RsnapNativeHost" SCROLL_COUNT="${SCROLL_COUNT:-14}" SCROLL_DRIVER="${SCROLL_DRIVER:-wheel}" SCROLL_START_METHOD="${SCROLL_START_METHOD:-keyboard}" @@ -114,6 +116,7 @@ export SCROLL_BACKGROUND_MODE SCROLL_BACKGROUND_PROOF_STRIPE restore_preferences() { live_hud_stop_awake_assertion live_hud_cancel_capture_if_present + stop_staged_rsnap_app if [[ -n "$SCROLL_BACKGROUND_PID" ]]; then kill "$SCROLL_BACKGROUND_PID" >/dev/null 2>&1 || true fi @@ -135,6 +138,32 @@ if defaults export "$PREF_DOMAIN" "$PREF_SNAPSHOT" >/dev/null 2>&1; then fi trap restore_preferences EXIT +stop_staged_rsnap_app() { + if [[ "${RSNAP_SMOKE_QUIT_APP_ON_EXIT:-1}" != "1" ]]; then + return 0 + fi + + local pids pid attempt + + pids="$(pgrep -f "$RSNAP_SMOKE_APP_EXECUTABLE" 2>/dev/null || true)" + if [[ -z "$pids" ]]; then + return 0 + fi + for pid in $pids; do + kill "$pid" >/dev/null 2>&1 || true + done + for attempt in {1..20}; do + pids="$(pgrep -f "$RSNAP_SMOKE_APP_EXECUTABLE" 2>/dev/null || true)" + if [[ -z "$pids" ]]; then + return 0 + fi + sleep 0.10 + done + for pid in $pids; do + kill -9 "$pid" >/dev/null 2>&1 || true + done +} + wait_ready_file() { local path="$1" local attempt From 233d240c980725575a85a1bd209df53ca1ab2bc2 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 12 May 2026 00:16:11 +0800 Subject: [PATCH 2/3] {"schema":"decodex/commit/1","summary":"Split scroll capture types and fingerprints","authority":"manual"} --- docs/reference/workspace-layout.md | 9 +- packages/rsnap-overlay/src/scroll_capture.rs | 416 +----------------- .../src/scroll_capture/fingerprint.rs | 79 ++++ .../src/scroll_capture/support.rs | 11 +- .../rsnap-overlay/src/scroll_capture/types.rs | 324 ++++++++++++++ 5 files changed, 431 insertions(+), 408 deletions(-) create mode 100644 packages/rsnap-overlay/src/scroll_capture/fingerprint.rs create mode 100644 packages/rsnap-overlay/src/scroll_capture/types.rs diff --git a/docs/reference/workspace-layout.md b/docs/reference/workspace-layout.md index 026ea97c..8a040823 100644 --- a/docs/reference/workspace-layout.md +++ b/docs/reference/workspace-layout.md @@ -96,8 +96,13 @@ Key paths: focused support modules under `scroll_capture/` - `packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs`: ordered worker-pairwise frame registration, committed-frontier catchup, rewind/reacquire handling, and growth-block decisions -- `packages/rsnap-overlay/src/scroll_capture/support.rs`: shared pixel matching, fingerprinting, - static-region rejection, and image-analysis helpers used by scroll capture +- `packages/rsnap-overlay/src/scroll_capture/types.rs`: scroll-capture data model, observation + outcomes, registration candidates, and telemetry structs shared by the session modules +- `packages/rsnap-overlay/src/scroll_capture/fingerprint.rs`: sampled frame fingerprinting used by + session duplicate detection and structural change tests +- `packages/rsnap-overlay/src/scroll_capture/support.rs`: shared pixel matching, + static-region rejection, image stacking/resizing, and image-analysis helpers used by scroll + capture - `packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs`: downward viewport candidate scoring and resolution helpers for session-owned stitching decisions diff --git a/packages/rsnap-overlay/src/scroll_capture.rs b/packages/rsnap-overlay/src/scroll_capture.rs index edfac02d..3ff73b72 100644 --- a/packages/rsnap-overlay/src/scroll_capture.rs +++ b/packages/rsnap-overlay/src/scroll_capture.rs @@ -1,12 +1,27 @@ pub mod bench_support; mod downward_resolution; +mod fingerprint; mod support; +mod types; mod worker_pairwise; +pub(crate) use self::fingerprint::ScrollFrameFingerprint; pub(crate) use self::support::{ compose_provisional_preview_image, scroll_capture_fingerprint, scroll_capture_fingerprint_delta, }; +#[cfg(test)] +pub(crate) use self::types::OverlapMatch; +pub(crate) use self::types::{ + BlockedPreviewOnlyLocalCandidate, CommittedDownwardViewportCandidateMode, DirectionMatch, + DirectionMatchEval, DownwardRegistration, DownwardRegistrationWithSource, DownwardSampleMatch, + DownwardSampleMatchSource, DownwardViewportCandidate, DownwardViewportCandidateSource, + DownwardViewportResolution, GrowthCommit, InformativeSpan, MotionObservation, + OverlapSearchConfig, OverlapSearchRange, PreviewOnlyDownwardLocalSample, + ResumeFrontierDirectMatchContext, ResumeFrontierMatchLog, ScrollCommitTelemetry, + ScrollDirection, ScrollObserveOutcome, UpInputMatchLog, UpInputSearchWindowLog, + UpwardInputDiagnostics, +}; use std::ops::RangeInclusive; @@ -21,8 +36,6 @@ pub(crate) const PREVIEW_ONLY_LOCAL_RECOVERY_MAX_TOLERANCE_ROWS: u32 = 12; pub(crate) const UNDERCONSUMED_OBSERVED_BURST_RECOVERY_GAP_ROWS: u32 = 8; pub(crate) const TRANSIENT_BURST_UNDERCONSUMED_HINT_MIN_ROWS: u32 = 48; -const FINGERPRINT_GRID_COLUMNS: u32 = 12; -const FINGERPRINT_GRID_ROWS: u32 = 16; const DOWNWARD_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 48; const DOWNWARD_KEYFRAME_SEARCH_MOTION_TOLERANCE_ROWS: u32 = 4; const DOWNWARD_KEYFRAME_SEARCH_MAX_TOLERANCE_ROWS: u32 = 48; @@ -48,150 +61,6 @@ const TRANSIENT_MOTION_HINT_MAX_MULTIPLIER: u32 = 3; const TRANSIENT_MOTION_HINT_MIN_CAP_ROWS: u32 = 12; const DIRECTION_WARNING_MARGIN_X100: u32 = 90; const RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100: u32 = 320; -const INFORMATIVE_SPAN_ROW_SAMPLES: u32 = 24; -const INFORMATIVE_SPAN_SCORE_FLOOR_X100: u32 = 24; -const INFORMATIVE_SPAN_HORIZONTAL_PADDING_PX: u32 = 16; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ScrollFrameFingerprint { - grid_columns: u32, - grid_rows: u32, - samples: Vec<[u8; 4]>, -} -impl ScrollFrameFingerprint { - #[must_use] - pub(crate) fn from_image(image: &RgbaImage) -> Self { - let width = image.width().max(1); - let height = image.height().max(1); - let informative_span = self::support::informative_column_span(image, 0, height); - let informative_left = - informative_span.map_or(0, |span| span.start_x.min(width.saturating_sub(1))); - let informative_right = informative_span - .map_or(width, |span| span.end_exclusive_x.min(width).max(informative_left + 1)); - let informative_width = informative_right.saturating_sub(informative_left).max(1); - let margin_x = ((informative_width as f32) * 0.05).round() as u32; - let margin_y = ((height as f32) * 0.05).round() as u32; - let left = - informative_left.saturating_add(margin_x).min(informative_right.saturating_sub(1)); - let right = informative_right.saturating_sub(margin_x).max(left + 1); - let top = margin_y.min(height.saturating_sub(1)); - let bottom = height.saturating_sub(margin_y).max(top + 1); - let mut samples = - Vec::with_capacity((FINGERPRINT_GRID_COLUMNS * FINGERPRINT_GRID_ROWS) as usize); - - for row in 0..FINGERPRINT_GRID_ROWS { - let y = self::support::evenly_spaced_sample(top, bottom, row, FINGERPRINT_GRID_ROWS); - - for column in 0..FINGERPRINT_GRID_COLUMNS { - let x = self::support::evenly_spaced_sample( - left, - right, - column, - FINGERPRINT_GRID_COLUMNS, - ); - let pixel = image.get_pixel(x, y).0; - - samples.push(pixel); - } - } - - Self { grid_columns: FINGERPRINT_GRID_COLUMNS, grid_rows: FINGERPRINT_GRID_ROWS, samples } - } - - #[must_use] - pub(crate) fn into_bytes(self) -> Vec { - let mut bytes = Vec::with_capacity(self.samples.len().saturating_mul(4)); - - for sample in self.samples { - bytes.extend_from_slice(&sample); - } - - bytes - } - - #[must_use] - #[cfg(test)] - pub(crate) fn distance(&self, other: &Self) -> u64 { - if self.grid_columns != other.grid_columns || self.grid_rows != other.grid_rows { - return u64::MAX; - } - - self.samples - .iter() - .zip(&other.samples) - .map(|(left, right)| { - u64::from(left[0].abs_diff(right[0])) - + u64::from(left[1].abs_diff(right[1])) - + u64::from(left[2].abs_diff(right[2])) - + u64::from(left[3].abs_diff(right[3])) - }) - .sum() - } -} - -#[cfg(test)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct OverlapMatch { - pub(crate) rows: u32, - pub(crate) matched: bool, - pub(crate) mean_abs_diff_x100: u32, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ScrollCommitTelemetry { - pub(crate) current_viewport_top_y: i32, - pub(crate) preview_dimensions: (u32, u32), - pub(crate) export_dimensions: (u32, u32), - pub(crate) growth_commit_count: usize, - pub(crate) preview_segment_count: usize, - pub(crate) export_segment_count: usize, - pub(crate) preview_export_segments_aligned: bool, - pub(crate) last_commit_decision_source: Option<&'static str>, - pub(crate) last_commit_detected_motion_rows: Option, - pub(crate) last_commit_effective_motion_rows_hint: Option, - pub(crate) last_block_reason: Option<&'static str>, - pub(crate) last_downward_sample_registration_result: Option<&'static str>, - pub(crate) last_downward_sample_registration_source: Option<&'static str>, - pub(crate) last_downward_sample_registration_motion_rows: Option, - pub(crate) last_downward_sample_registration_provisional_viewport_top_y: Option, - pub(crate) observed_sample_registration_result: Option<&'static str>, - pub(crate) observed_sample_registration_reason: Option<&'static str>, - pub(crate) observed_sample_registration_motion_rows: Option, - pub(crate) observed_sample_registration_mean_abs_diff_x100: Option, - pub(crate) preview_only_local_registration_result: Option<&'static str>, - pub(crate) preview_only_local_registration_reason: Option<&'static str>, - pub(crate) preview_only_local_registration_motion_rows: Option, - pub(crate) preview_only_local_registration_mean_abs_diff_x100: Option, - pub(crate) last_downward_viewport_candidate_count: Option, - pub(crate) last_downward_viewport_candidates_before_prune: Option, - pub(crate) last_downward_viewport_candidates_after_prune: Option, - pub(crate) sample_eval_last_motion_rows_hint: Option, - pub(crate) sample_eval_transient_motion_rows_hint: Option, - pub(crate) sample_eval_effective_motion_rows_hint: Option, - pub(crate) sample_eval_transient_burst_search_enabled: bool, - pub(crate) preview_only_local_viewport_top_y: Option, - pub(crate) last_preview_segment_height_px: Option, - pub(crate) last_export_segment_height_px: Option, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct OverlapSearchConfig { - pub(crate) min_overlap_rows: u32, - pub(crate) max_column_samples: u32, - pub(crate) max_row_samples: u32, - pub(crate) max_mean_abs_diff_x100: u32, -} -impl Default for OverlapSearchConfig { - fn default() -> Self { - Self { - min_overlap_rows: 24, - max_column_samples: 160, - max_row_samples: 64, - max_mean_abs_diff_x100: 850, - } - } -} - #[derive(Clone, Debug)] pub(crate) struct ScrollSession { anchor_frame: RgbaImage, @@ -2955,261 +2824,6 @@ impl ScrollSession { } } -#[derive(Clone, Debug, Eq, PartialEq)] -struct PreviewOnlyDownwardLocalSample { - frame: RgbaImage, - viewport_top_y: i32, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectionMatch { - mean_abs_diff_x100: u32, - motion_rows: u32, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DownwardSampleMatch { - matched: DirectionMatch, - source: DownwardSampleMatchSource, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DownwardViewportCandidate { - source: DownwardViewportCandidateSource, - viewport_top_y: i32, - motion_rows: u32, - mean_abs_diff_x100: u32, -} -impl DownwardViewportCandidate { - fn competing_block_reason(self, competing: Self) -> &'static str { - match (self.source, competing.source) { - ( - DownwardViewportCandidateSource::CommittedKeyframe, - DownwardViewportCandidateSource::CommittedKeyframe, - ) => "conflicting_committed_keyframe_authority", - _ => "conflicting_downward_viewport_authority", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct BlockedPreviewOnlyLocalCandidate { - candidate: DownwardViewportCandidate, - repeats: u8, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct OverlapSearchRange { - start: u32, - end: u32, -} -impl OverlapSearchRange { - fn as_range(self) -> RangeInclusive { - self.start..=self.end - } -} - -impl From> for OverlapSearchRange { - fn from(range: RangeInclusive) -> Self { - Self { start: *range.start(), end: *range.end() } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectionMatchEval { - preferred_range: Option, - max_motion_rows: u32, - preferred_only_match: Option, - final_match: Option, - used_full_range_fallback: bool, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct MotionObservation { - direction: ScrollDirection, - motion_rows: u32, -} - -#[derive(Clone, Copy, Debug)] -struct UpInputMatchLog { - sample_motion: Option, - sample_down_match: Option, - sample_up_match: Option, - committed_down_match: Option, - committed_up_match: Option, - sample_override_wins: bool, - committed_override_wins: bool, -} - -#[derive(Clone, Copy, Debug)] -struct UpInputSearchWindowLog<'a> { - sample_delta: Option, - sample_down_match_eval: &'a DirectionMatchEval, - sample_up_match_eval: &'a DirectionMatchEval, - committed_down_match_eval: &'a DirectionMatchEval, - committed_up_match_eval: &'a DirectionMatchEval, - frame_equals_last_sample: bool, - frame_equals_last_committed: bool, -} - -#[derive(Clone, Copy, Debug)] -struct UpwardInputDiagnostics { - sample_down_match_eval: DirectionMatchEval, - sample_up_match_eval: DirectionMatchEval, - committed_down_match_eval: DirectionMatchEval, - committed_up_match_eval: DirectionMatchEval, - sample_override_match: Option, - committed_override_match: Option, -} - -#[derive(Clone, Copy, Debug)] -struct ResumeFrontierMatchLog { - motion_rows: u32, - candidate_observed_viewport_top_y: i32, - residual_growth_rows: u32, - raw_committed_down_match: Option, - trusted_committed_down_match: Option, - committed_up_match: Option, - frame_reacquires_last_committed_viewport: bool, -} - -#[derive(Clone, Copy, Debug)] -struct ResumeFrontierDirectMatchContext { - motion_rows: u32, - candidate_observed_viewport_top_y: i32, - residual_growth_rows: u32, -} - -#[derive(Clone, Debug)] -struct GrowthCommit { - frame: RgbaImage, - growth_rows: u32, - viewport_top_y: i32, - decision_source: &'static str, - detected_motion_rows: Option, - effective_motion_rows_hint: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct InformativeSpan { - start_x: u32, - end_exclusive_x: u32, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ScrollDirection { - Up, - Down, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ScrollObserveOutcome { - NoChange, - PreviewUpdated, - UnsupportedDirection { direction: ScrollDirection }, - Committed { direction: ScrollDirection, growth_rows: u32 }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardRegistration { - NoMatch, - Matched(DirectionMatch), - Ambiguous { best: DirectionMatch, competing: DirectionMatch }, -} -impl DownwardRegistration { - fn map_source(self, source: DownwardSampleMatchSource) -> DownwardRegistrationWithSource { - match self { - Self::NoMatch => DownwardRegistrationWithSource::NoMatch, - Self::Matched(matched) => { - DownwardRegistrationWithSource::Matched(DownwardSampleMatch { matched, source }) - }, - Self::Ambiguous { best, competing } => DownwardRegistrationWithSource::Ambiguous { - best: DownwardSampleMatch { matched: best, source }, - competing: DownwardSampleMatch { matched: competing, source }, - }, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardSampleMatchSource { - ObservedSample, - PreviewOnlyLocalSample, -} -impl DownwardSampleMatchSource { - const fn label(self) -> &'static str { - match self { - Self::ObservedSample => "observed_sample", - Self::PreviewOnlyLocalSample => "preview_only_local_sample", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardRegistrationWithSource { - NoMatch, - Matched(DownwardSampleMatch), - Ambiguous { best: DownwardSampleMatch, competing: DownwardSampleMatch }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardViewportCandidateSource { - ObservedSample, - PreviewOnlyLocalSample, - CommittedKeyframe, -} -impl DownwardViewportCandidateSource { - const fn priority(self) -> u8 { - match self { - Self::CommittedKeyframe => 0, - Self::ObservedSample => 1, - Self::PreviewOnlyLocalSample => 2, - } - } - - const fn decision_source(self) -> &'static str { - match self { - Self::ObservedSample => "sample_motion_downward_growth_from_observed_keyframe", - Self::PreviewOnlyLocalSample => { - "sample_motion_downward_growth_from_preview_only_local_sample" - }, - Self::CommittedKeyframe => "sample_motion_downward_growth_from_committed_keyframe", - } - } - - const fn fallback_decision_source(self) -> &'static str { - match self { - Self::ObservedSample => "fallback_downward_registration_from_observed_keyframe", - Self::PreviewOnlyLocalSample => { - "fallback_downward_registration_from_preview_only_local_sample" - }, - Self::CommittedKeyframe => "fallback_downward_registration_from_committed_keyframe", - } - } -} - -impl From for DownwardViewportCandidateSource { - fn from(value: DownwardSampleMatchSource) -> Self { - match value { - DownwardSampleMatchSource::ObservedSample => Self::ObservedSample, - DownwardSampleMatchSource::PreviewOnlyLocalSample => Self::PreviewOnlyLocalSample, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum CommittedDownwardViewportCandidateMode { - LastCommittedOnly, - IncludeRecentHistory, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum DownwardViewportResolution { - NoMatch, - Selected(DownwardViewportCandidate), - Ambiguous { preferred: DownwardViewportCandidate, competing: DownwardViewportCandidate }, -} - #[cfg(test)] pub(crate) mod test_support; diff --git a/packages/rsnap-overlay/src/scroll_capture/fingerprint.rs b/packages/rsnap-overlay/src/scroll_capture/fingerprint.rs new file mode 100644 index 00000000..407ee561 --- /dev/null +++ b/packages/rsnap-overlay/src/scroll_capture/fingerprint.rs @@ -0,0 +1,79 @@ +use image::RgbaImage; + +use crate::scroll_capture::support; + +const FINGERPRINT_GRID_COLUMNS: u32 = 12; +const FINGERPRINT_GRID_ROWS: u32 = 16; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ScrollFrameFingerprint { + grid_columns: u32, + grid_rows: u32, + samples: Vec<[u8; 4]>, +} +impl ScrollFrameFingerprint { + #[must_use] + pub(crate) fn from_image(image: &RgbaImage) -> Self { + let width = image.width().max(1); + let height = image.height().max(1); + let informative_span = support::informative_column_span(image, 0, height); + let informative_left = + informative_span.map_or(0, |span| span.start_x.min(width.saturating_sub(1))); + let informative_right = informative_span + .map_or(width, |span| span.end_exclusive_x.min(width).max(informative_left + 1)); + let informative_width = informative_right.saturating_sub(informative_left).max(1); + let margin_x = ((informative_width as f32) * 0.05).round() as u32; + let margin_y = ((height as f32) * 0.05).round() as u32; + let left = + informative_left.saturating_add(margin_x).min(informative_right.saturating_sub(1)); + let right = informative_right.saturating_sub(margin_x).max(left + 1); + let top = margin_y.min(height.saturating_sub(1)); + let bottom = height.saturating_sub(margin_y).max(top + 1); + let mut samples = + Vec::with_capacity((FINGERPRINT_GRID_COLUMNS * FINGERPRINT_GRID_ROWS) as usize); + + for row in 0..FINGERPRINT_GRID_ROWS { + let y = support::evenly_spaced_sample(top, bottom, row, FINGERPRINT_GRID_ROWS); + + for column in 0..FINGERPRINT_GRID_COLUMNS { + let x = + support::evenly_spaced_sample(left, right, column, FINGERPRINT_GRID_COLUMNS); + let pixel = image.get_pixel(x, y).0; + + samples.push(pixel); + } + } + + Self { grid_columns: FINGERPRINT_GRID_COLUMNS, grid_rows: FINGERPRINT_GRID_ROWS, samples } + } + + #[must_use] + pub(crate) fn into_bytes(self) -> Vec { + let mut bytes = Vec::with_capacity(self.samples.len().saturating_mul(4)); + + for sample in self.samples { + bytes.extend_from_slice(&sample); + } + + bytes + } + + #[must_use] + #[cfg(test)] + pub(crate) fn distance(&self, other: &Self) -> u64 { + if self.grid_columns != other.grid_columns || self.grid_rows != other.grid_rows { + return u64::MAX; + } + + self.samples + .iter() + .zip(&other.samples) + .map(|(left, right)| { + u64::from(left[0].abs_diff(right[0])) + + u64::from(left[1].abs_diff(right[1])) + + u64::from(left[2].abs_diff(right[2])) + + u64::from(left[3].abs_diff(right[3])) + }) + .sum() + } +} diff --git a/packages/rsnap-overlay/src/scroll_capture/support.rs b/packages/rsnap-overlay/src/scroll_capture/support.rs index b38417d3..c9573f74 100644 --- a/packages/rsnap-overlay/src/scroll_capture/support.rs +++ b/packages/rsnap-overlay/src/scroll_capture/support.rs @@ -29,13 +29,14 @@ use crate::scroll_capture::{ DIRECTION_WARNING_MARGIN_X100, DOWNWARD_REGISTRATION_AMBIGUOUS_GAP_ROWS, DOWNWARD_REGISTRATION_MIN_OVERLAP_DIVISOR, DOWNWARD_VIEWPORT_AUTHORITY_GAP_ROWS, DirectionMatch, DownwardRegistration, DownwardViewportCandidate, - DownwardViewportCandidateSource, DownwardViewportResolution, - INFORMATIVE_SPAN_HORIZONTAL_PADDING_PX, INFORMATIVE_SPAN_ROW_SAMPLES, - INFORMATIVE_SPAN_SCORE_FLOOR_X100, InformativeSpan, OverlapSearchConfig, - RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100, ScrollDirection, ScrollFrameFingerprint, - ScrollObserveOutcome, + DownwardViewportCandidateSource, DownwardViewportResolution, InformativeSpan, + OverlapSearchConfig, RESUME_DIRECT_PROOF_MAX_MEAN_ABS_DIFF_X100, ScrollDirection, + ScrollFrameFingerprint, ScrollObserveOutcome, }; +const INFORMATIVE_SPAN_ROW_SAMPLES: u32 = 24; +const INFORMATIVE_SPAN_SCORE_FLOOR_X100: u32 = 24; +const INFORMATIVE_SPAN_HORIZONTAL_PADDING_PX: u32 = 16; const MOTION_COVERAGE_MIN_PERCENT: u32 = 20; const MOTION_COVERAGE_MIN_INFORMATIVE_COLUMNS: u32 = 1; const MOTION_COVERAGE_STATIC_EDGE_MAX_LEADING_COLUMNS: u32 = 48; diff --git a/packages/rsnap-overlay/src/scroll_capture/types.rs b/packages/rsnap-overlay/src/scroll_capture/types.rs new file mode 100644 index 00000000..945cdfe6 --- /dev/null +++ b/packages/rsnap-overlay/src/scroll_capture/types.rs @@ -0,0 +1,324 @@ +use std::ops::RangeInclusive; + +use image::RgbaImage; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct OverlapMatch { + pub(crate) rows: u32, + pub(crate) matched: bool, + pub(crate) mean_abs_diff_x100: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ScrollCommitTelemetry { + pub(crate) current_viewport_top_y: i32, + pub(crate) preview_dimensions: (u32, u32), + pub(crate) export_dimensions: (u32, u32), + pub(crate) growth_commit_count: usize, + pub(crate) preview_segment_count: usize, + pub(crate) export_segment_count: usize, + pub(crate) preview_export_segments_aligned: bool, + pub(crate) last_commit_decision_source: Option<&'static str>, + pub(crate) last_commit_detected_motion_rows: Option, + pub(crate) last_commit_effective_motion_rows_hint: Option, + pub(crate) last_block_reason: Option<&'static str>, + pub(crate) last_downward_sample_registration_result: Option<&'static str>, + pub(crate) last_downward_sample_registration_source: Option<&'static str>, + pub(crate) last_downward_sample_registration_motion_rows: Option, + pub(crate) last_downward_sample_registration_provisional_viewport_top_y: Option, + pub(crate) observed_sample_registration_result: Option<&'static str>, + pub(crate) observed_sample_registration_reason: Option<&'static str>, + pub(crate) observed_sample_registration_motion_rows: Option, + pub(crate) observed_sample_registration_mean_abs_diff_x100: Option, + pub(crate) preview_only_local_registration_result: Option<&'static str>, + pub(crate) preview_only_local_registration_reason: Option<&'static str>, + pub(crate) preview_only_local_registration_motion_rows: Option, + pub(crate) preview_only_local_registration_mean_abs_diff_x100: Option, + pub(crate) last_downward_viewport_candidate_count: Option, + pub(crate) last_downward_viewport_candidates_before_prune: Option, + pub(crate) last_downward_viewport_candidates_after_prune: Option, + pub(crate) sample_eval_last_motion_rows_hint: Option, + pub(crate) sample_eval_transient_motion_rows_hint: Option, + pub(crate) sample_eval_effective_motion_rows_hint: Option, + pub(crate) sample_eval_transient_burst_search_enabled: bool, + pub(crate) preview_only_local_viewport_top_y: Option, + pub(crate) last_preview_segment_height_px: Option, + pub(crate) last_export_segment_height_px: Option, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct OverlapSearchConfig { + pub(crate) min_overlap_rows: u32, + pub(crate) max_column_samples: u32, + pub(crate) max_row_samples: u32, + pub(crate) max_mean_abs_diff_x100: u32, +} +impl Default for OverlapSearchConfig { + fn default() -> Self { + Self { + min_overlap_rows: 24, + max_column_samples: 160, + max_row_samples: 64, + max_mean_abs_diff_x100: 850, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PreviewOnlyDownwardLocalSample { + pub(crate) frame: RgbaImage, + pub(crate) viewport_top_y: i32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct DirectionMatch { + pub(crate) mean_abs_diff_x100: u32, + pub(crate) motion_rows: u32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct DownwardSampleMatch { + pub(crate) matched: DirectionMatch, + pub(crate) source: DownwardSampleMatchSource, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct DownwardViewportCandidate { + pub(crate) source: DownwardViewportCandidateSource, + pub(crate) viewport_top_y: i32, + pub(crate) motion_rows: u32, + pub(crate) mean_abs_diff_x100: u32, +} +impl DownwardViewportCandidate { + pub(crate) fn competing_block_reason(self, competing: Self) -> &'static str { + match (self.source, competing.source) { + ( + DownwardViewportCandidateSource::CommittedKeyframe, + DownwardViewportCandidateSource::CommittedKeyframe, + ) => "conflicting_committed_keyframe_authority", + _ => "conflicting_downward_viewport_authority", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct BlockedPreviewOnlyLocalCandidate { + pub(crate) candidate: DownwardViewportCandidate, + pub(crate) repeats: u8, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct OverlapSearchRange { + pub(crate) start: u32, + pub(crate) end: u32, +} +impl OverlapSearchRange { + pub(crate) fn as_range(self) -> RangeInclusive { + self.start..=self.end + } +} + +impl From> for OverlapSearchRange { + fn from(range: RangeInclusive) -> Self { + Self { start: *range.start(), end: *range.end() } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct DirectionMatchEval { + pub(crate) preferred_range: Option, + pub(crate) max_motion_rows: u32, + pub(crate) preferred_only_match: Option, + pub(crate) final_match: Option, + pub(crate) used_full_range_fallback: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct MotionObservation { + pub(crate) direction: ScrollDirection, + pub(crate) motion_rows: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct UpInputMatchLog { + pub(crate) sample_motion: Option, + pub(crate) sample_down_match: Option, + pub(crate) sample_up_match: Option, + pub(crate) committed_down_match: Option, + pub(crate) committed_up_match: Option, + pub(crate) sample_override_wins: bool, + pub(crate) committed_override_wins: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct UpInputSearchWindowLog<'a> { + pub(crate) sample_delta: Option, + pub(crate) sample_down_match_eval: &'a DirectionMatchEval, + pub(crate) sample_up_match_eval: &'a DirectionMatchEval, + pub(crate) committed_down_match_eval: &'a DirectionMatchEval, + pub(crate) committed_up_match_eval: &'a DirectionMatchEval, + pub(crate) frame_equals_last_sample: bool, + pub(crate) frame_equals_last_committed: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct UpwardInputDiagnostics { + pub(crate) sample_down_match_eval: DirectionMatchEval, + pub(crate) sample_up_match_eval: DirectionMatchEval, + pub(crate) committed_down_match_eval: DirectionMatchEval, + pub(crate) committed_up_match_eval: DirectionMatchEval, + pub(crate) sample_override_match: Option, + pub(crate) committed_override_match: Option, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ResumeFrontierMatchLog { + pub(crate) motion_rows: u32, + pub(crate) candidate_observed_viewport_top_y: i32, + pub(crate) residual_growth_rows: u32, + pub(crate) raw_committed_down_match: Option, + pub(crate) trusted_committed_down_match: Option, + pub(crate) committed_up_match: Option, + pub(crate) frame_reacquires_last_committed_viewport: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ResumeFrontierDirectMatchContext { + pub(crate) motion_rows: u32, + pub(crate) candidate_observed_viewport_top_y: i32, + pub(crate) residual_growth_rows: u32, +} + +#[derive(Clone, Debug)] +pub(crate) struct GrowthCommit { + pub(crate) frame: RgbaImage, + pub(crate) growth_rows: u32, + pub(crate) viewport_top_y: i32, + pub(crate) decision_source: &'static str, + pub(crate) detected_motion_rows: Option, + pub(crate) effective_motion_rows_hint: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct InformativeSpan { + pub(crate) start_x: u32, + pub(crate) end_exclusive_x: u32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ScrollDirection { + Up, + Down, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ScrollObserveOutcome { + NoChange, + PreviewUpdated, + UnsupportedDirection { direction: ScrollDirection }, + Committed { direction: ScrollDirection, growth_rows: u32 }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DownwardRegistration { + NoMatch, + Matched(DirectionMatch), + Ambiguous { best: DirectionMatch, competing: DirectionMatch }, +} +impl DownwardRegistration { + pub(crate) fn map_source( + self, + source: DownwardSampleMatchSource, + ) -> DownwardRegistrationWithSource { + match self { + Self::NoMatch => DownwardRegistrationWithSource::NoMatch, + Self::Matched(matched) => { + DownwardRegistrationWithSource::Matched(DownwardSampleMatch { matched, source }) + }, + Self::Ambiguous { best, competing } => DownwardRegistrationWithSource::Ambiguous { + best: DownwardSampleMatch { matched: best, source }, + competing: DownwardSampleMatch { matched: competing, source }, + }, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DownwardSampleMatchSource { + ObservedSample, + PreviewOnlyLocalSample, +} +impl DownwardSampleMatchSource { + pub(crate) const fn label(self) -> &'static str { + match self { + Self::ObservedSample => "observed_sample", + Self::PreviewOnlyLocalSample => "preview_only_local_sample", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DownwardRegistrationWithSource { + NoMatch, + Matched(DownwardSampleMatch), + Ambiguous { best: DownwardSampleMatch, competing: DownwardSampleMatch }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DownwardViewportCandidateSource { + ObservedSample, + PreviewOnlyLocalSample, + CommittedKeyframe, +} +impl DownwardViewportCandidateSource { + pub(crate) const fn priority(self) -> u8 { + match self { + Self::CommittedKeyframe => 0, + Self::ObservedSample => 1, + Self::PreviewOnlyLocalSample => 2, + } + } + + pub(crate) const fn decision_source(self) -> &'static str { + match self { + Self::ObservedSample => "sample_motion_downward_growth_from_observed_keyframe", + Self::PreviewOnlyLocalSample => { + "sample_motion_downward_growth_from_preview_only_local_sample" + }, + Self::CommittedKeyframe => "sample_motion_downward_growth_from_committed_keyframe", + } + } + + pub(crate) const fn fallback_decision_source(self) -> &'static str { + match self { + Self::ObservedSample => "fallback_downward_registration_from_observed_keyframe", + Self::PreviewOnlyLocalSample => { + "fallback_downward_registration_from_preview_only_local_sample" + }, + Self::CommittedKeyframe => "fallback_downward_registration_from_committed_keyframe", + } + } +} + +impl From for DownwardViewportCandidateSource { + fn from(value: DownwardSampleMatchSource) -> Self { + match value { + DownwardSampleMatchSource::ObservedSample => Self::ObservedSample, + DownwardSampleMatchSource::PreviewOnlyLocalSample => Self::PreviewOnlyLocalSample, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CommittedDownwardViewportCandidateMode { + LastCommittedOnly, + IncludeRecentHistory, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DownwardViewportResolution { + NoMatch, + Selected(DownwardViewportCandidate), + Ambiguous { preferred: DownwardViewportCandidate, competing: DownwardViewportCandidate }, +} From 07ce598c8e769ba4eac7f1863521e77689d3aa83 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 12 May 2026 01:08:34 +0800 Subject: [PATCH 3/3] {"schema":"decodex/commit/1","summary":"Add scroll toolbar glass cadence checks","authority":"manual"} --- .../RsnapNativeHostKit/CaptureHostView.swift | 16 +++++ scripts/smoke/native-scroll-capture-macos.sh | 62 ++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift b/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift index d3fac39e..d5b8a375 100644 --- a/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift +++ b/native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift @@ -290,6 +290,16 @@ final class CaptureHostView: NSView { private var frozenToolbarLiquidGlassVisible = false private var frozenToolbarLiquidGlassContentDrawn = false private var lastScrollCaptureToolbarBackdropRefreshUptime: TimeInterval = 0 + private let scrollToolbarBackdropRefreshGapMetric = NativeHostTelemetry.distribution( + "scroll_capture.toolbar_backdrop_refresh_gap", + category: "Capture", + batchSize: 30 + ) + private let scrollToolbarBackdropRefreshDurationMetric = NativeHostTelemetry.distribution( + "scroll_capture.toolbar_backdrop_refresh_duration", + category: "Capture", + batchSize: 30 + ) private var trackingAreaRef: NSTrackingArea? private var pointerOverFrozenToolbar = false private var hoveredToolbarAction: ToolbarItemKind? @@ -3568,8 +3578,14 @@ final class CaptureHostView: NSView { guard now - lastScrollCaptureToolbarBackdropRefreshUptime >= interval else { return } + if lastScrollCaptureToolbarBackdropRefreshUptime > 0 { + scrollToolbarBackdropRefreshGapMetric.record( + (now - lastScrollCaptureToolbarBackdropRefreshUptime) * 1_000) + } lastScrollCaptureToolbarBackdropRefreshUptime = now + let refreshStartedAt = now updateFrozenToolbarLiquidGlassView() + scrollToolbarBackdropRefreshDurationMetric.recordMillisecondsSince(refreshStartedAt) } private func localAnnotationStyleLayout( diff --git a/scripts/smoke/native-scroll-capture-macos.sh b/scripts/smoke/native-scroll-capture-macos.sh index 51ee2f00..deb7e0ea 100755 --- a/scripts/smoke/native-scroll-capture-macos.sh +++ b/scripts/smoke/native-scroll-capture-macos.sh @@ -26,6 +26,8 @@ Useful overrides: EXPECT_SCROLL_GROWTH=1 defaults to 0 for codex_like MIN_SCROLL_COMMITS=2 defaults to 0 when EXPECT_SCROLL_GROWTH=0 MIN_EXPORT_GROWTH_PX=180 defaults to 0 when EXPECT_SCROLL_GROWTH=0 + MAX_SCROLL_TOOLBAR_BACKDROP_GAP_P50_MS=24 + MAX_SCROLL_TOOLBAR_BACKDROP_DURATION_P95_MS=8 VALIDATE_SCROLL_EXPORT_CONTINUITY=0 set to 1 with SCROLL_BACKGROUND_PROOF_STRIPE=1 APP_POST_VERIFY_SETTLE_S=0 POST_FREEZE_SETTLE_S=2.0 @@ -97,6 +99,8 @@ if [[ -z "${MIN_EXPORT_GROWTH_PX:-}" ]]; then fi fi APP_POST_VERIFY_SETTLE_S="${APP_POST_VERIFY_SETTLE_S:-0}" +MAX_SCROLL_TOOLBAR_BACKDROP_GAP_P50_MS="${MAX_SCROLL_TOOLBAR_BACKDROP_GAP_P50_MS:-24}" +MAX_SCROLL_TOOLBAR_BACKDROP_DURATION_P95_MS="${MAX_SCROLL_TOOLBAR_BACKDROP_DURATION_P95_MS:-8}" VALIDATE_SCROLL_EXPORT_CONTINUITY="${VALIDATE_SCROLL_EXPORT_CONTINUITY:-0}" OVERLAY_SETTLE_S="${OVERLAY_SETTLE_S:-0.10}" POST_FREEZE_SETTLE_S="${POST_FREEZE_SETTLE_S:-2.0}" @@ -303,14 +307,24 @@ start_scroll_background() { parse_telemetry() { local log_path="$1" - python3 - "$log_path" "$MIN_SCROLL_COMMITS" "$MIN_EXPORT_GROWTH_PX" "$SCROLL_START_METHOD" "$EXPECT_SCROLL_GROWTH" <<'PY' + python3 - "$log_path" "$MIN_SCROLL_COMMITS" "$MIN_EXPORT_GROWTH_PX" "$SCROLL_START_METHOD" "$EXPECT_SCROLL_GROWTH" "$MAX_SCROLL_TOOLBAR_BACKDROP_GAP_P50_MS" "$MAX_SCROLL_TOOLBAR_BACKDROP_DURATION_P95_MS" <<'PY' import re import sys -path, min_commits_raw, min_growth_raw, start_method, expect_growth_raw = sys.argv[1:6] +( + path, + min_commits_raw, + min_growth_raw, + start_method, + expect_growth_raw, + max_toolbar_gap_p50_raw, + max_toolbar_duration_p95_raw, +) = sys.argv[1:8] min_commits = int(min_commits_raw) min_growth = int(min_growth_raw) expect_growth = expect_growth_raw != "0" +max_toolbar_gap_p50 = float(max_toolbar_gap_p50_raw) +max_toolbar_duration_p95 = float(max_toolbar_duration_p95_raw) expected_start_source = { "keyboard": "keyboard_s", "toolbar": "toolbar", @@ -369,6 +383,29 @@ missing = "event=capture.scroll_sample_missing" in text auto_event = bool(re.search(r"event=capture\.scroll_auto_", text)) max_height = max(heights) if heights else 0 growth = max_height - base_height +toolbar_glass_expected = ( + "usesLiquidHudGlass=true" in text + and "frozenToolbarLiquidGlassVisible=true" in text +) +toolbar_gap_metrics = [ + (int(samples), float(p50), float(p95), float(max_value)) + for samples, p50, p95, max_value in re.findall( + r"metric=scroll_capture\.toolbar_backdrop_refresh_gap\b[^\n]*samples=([0-9]+)[^\n]*p50=([0-9.]+)[^\n]*p95=([0-9.]+)[^\n]*max=([0-9.]+)", + text, + ) +] +toolbar_duration_metrics = [ + (int(samples), float(p50), float(p95), float(max_value)) + for samples, p50, p95, max_value in re.findall( + r"metric=scroll_capture\.toolbar_backdrop_refresh_duration\b[^\n]*samples=([0-9]+)[^\n]*p50=([0-9.]+)[^\n]*p95=([0-9.]+)[^\n]*max=([0-9.]+)", + text, + ) +] +toolbar_gap_samples = sum(samples for samples, _, _, _ in toolbar_gap_metrics) +toolbar_duration_samples = sum(samples for samples, _, _, _ in toolbar_duration_metrics) +toolbar_gap_p50 = max((p50 for _, p50, _, _ in toolbar_gap_metrics), default=None) +toolbar_gap_p95 = max((p95 for _, _, p95, _ in toolbar_gap_metrics), default=None) +toolbar_duration_p95 = max((p95 for _, _, p95, _ in toolbar_duration_metrics), default=None) print( f"[smoke] telemetry froze={froze} handoff={handoff} " @@ -378,7 +415,13 @@ print( f"tap_not_used={tap_not_used} wheel_intercepted={wheel_intercepted} " f"wheel_observed={wheel_observed} " f"max_export_height={max_height} base_height={base_height} growth={growth} " - f"missing_live_frame={missing} auto_event={auto_event}" + f"missing_live_frame={missing} auto_event={auto_event} " + f"toolbar_glass_expected={toolbar_glass_expected} " + f"toolbar_backdrop_gap_samples={toolbar_gap_samples} " + f"toolbar_backdrop_gap_p50={toolbar_gap_p50 if toolbar_gap_p50 is not None else 'none'} " + f"toolbar_backdrop_gap_p95={toolbar_gap_p95 if toolbar_gap_p95 is not None else 'none'} " + f"toolbar_backdrop_duration_samples={toolbar_duration_samples} " + f"toolbar_backdrop_duration_p95={toolbar_duration_p95 if toolbar_duration_p95 is not None else 'none'}" ) failures = [] @@ -414,6 +457,19 @@ else: failures.append(f"expected fail-closed no export growth, but growth was {growth}px") if auto_event: failures.append("unexpected legacy auto-scroll telemetry") +if toolbar_glass_expected: + if toolbar_gap_p50 is None: + failures.append("scroll toolbar liquid-glass backdrop refresh gap metric was not emitted") + elif toolbar_gap_p50 > max_toolbar_gap_p50: + failures.append( + f"scroll toolbar backdrop refresh gap p50 {toolbar_gap_p50:.2f}ms > {max_toolbar_gap_p50:.2f}ms" + ) + if toolbar_duration_p95 is None: + failures.append("scroll toolbar liquid-glass backdrop refresh duration metric was not emitted") + elif toolbar_duration_p95 > max_toolbar_duration_p95: + failures.append( + f"scroll toolbar backdrop refresh duration p95 {toolbar_duration_p95:.2f}ms > {max_toolbar_duration_p95:.2f}ms" + ) if failures: for failure in failures: