Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/reference/host-core-reset.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ The current native-host Swift split is:
lifecycle hooks; the extensions split live capture/input, frozen selection interactions,
host-request draining, native scroll-capture sampling, copy/save/export effects, Vision OCR, and
runtime teardown/window helpers.
- `NativeScrollCaptureObservationPipeline.swift`: native scroll-capture sample batching, fallback
sample adaptation, Rust scroll-observation calls, and preview export refresh packaging. It keeps
ordered frame acquisition and AppKit scheduling in Swift while leaving stitching decisions in
Rust.
- `CaptureChrome.swift`: shared native chrome metrics, palette, dashed-border geometry, and
AppKit color/image helpers used by live and frozen capture UI.
- `CaptureOverlayWindow.swift`: the AppKit `NSPanel` wrapper that embeds `CaptureHostView` for each
Expand Down
13 changes: 13 additions & 0 deletions docs/reference/workspace-layout.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,17 @@ Key paths:
- `packages/rsnap-overlay/src/live_frame_stream_macos.rs`: current macOS live-stream support
- `packages/rsnap-overlay/src/scroll_capture.rs`: current scroll-capture session entry with
focused support modules under `scroll_capture/`
- `packages/rsnap-overlay/src/scroll_capture/worker_pairwise.rs`: ordered worker-pairwise frame
registration, committed-frontier catchup, rewind/reacquire handling, and growth-block decisions
- `packages/rsnap-overlay/src/scroll_capture/types.rs`: scroll-capture data model, observation
outcomes, registration candidates, and telemetry structs shared by the session modules
- `packages/rsnap-overlay/src/scroll_capture/fingerprint.rs`: sampled frame fingerprinting used by
session duplicate detection and structural change tests
- `packages/rsnap-overlay/src/scroll_capture/support.rs`: shared pixel matching,
static-region rejection, image stacking/resizing, and image-analysis helpers used by scroll
capture
- `packages/rsnap-overlay/src/scroll_capture/downward_resolution.rs`: downward viewport candidate
scoring and resolution helpers for session-owned stitching decisions

### `packages/rsnap-capture-core/`

Expand Down Expand Up @@ -165,6 +176,8 @@ The main host-kit files are split by responsibility:
commit handling, host-owned frozen scene preparation, and host-effect dispatch
- `CaptureSessionController+ScrollCapture.swift`: native scroll monitor lifecycle, scroll-event
forwarding, viewport sampling, and scroll minimap preview refresh
- `NativeScrollCaptureObservationPipeline.swift`: conversion of ordered native samples and
fallback frames into Rust scroll observations plus preview export batches
- `CaptureSessionController+Export.swift`: copy/save host effects, output naming, capture-image
export, capture-frame effect application, and Rust-backed PNG encoding
- `CaptureSessionController+TextRecognition.swift`: Vision OCR request execution and recognized
Expand Down
16 changes: 16 additions & 0 deletions native/macos-host/Sources/RsnapNativeHostKit/CaptureHostView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,16 @@ final class CaptureHostView: NSView {
private var frozenToolbarLiquidGlassVisible = false
private var frozenToolbarLiquidGlassContentDrawn = false
private var lastScrollCaptureToolbarBackdropRefreshUptime: TimeInterval = 0
private let scrollToolbarBackdropRefreshGapMetric = NativeHostTelemetry.distribution(
"scroll_capture.toolbar_backdrop_refresh_gap",
category: "Capture",
batchSize: 30
)
private let scrollToolbarBackdropRefreshDurationMetric = NativeHostTelemetry.distribution(
"scroll_capture.toolbar_backdrop_refresh_duration",
category: "Capture",
batchSize: 30
)
private var trackingAreaRef: NSTrackingArea?
private var pointerOverFrozenToolbar = false
private var hoveredToolbarAction: ToolbarItemKind?
Expand Down Expand Up @@ -3568,8 +3578,14 @@ final class CaptureHostView: NSView {
guard now - lastScrollCaptureToolbarBackdropRefreshUptime >= interval else {
return
}
if lastScrollCaptureToolbarBackdropRefreshUptime > 0 {
scrollToolbarBackdropRefreshGapMetric.record(
(now - lastScrollCaptureToolbarBackdropRefreshUptime) * 1_000)
}
lastScrollCaptureToolbarBackdropRefreshUptime = now
let refreshStartedAt = now
updateFrozenToolbarLiquidGlassView()
scrollToolbarBackdropRefreshDurationMetric.recordMillisecondsSince(refreshStartedAt)
}

private func localAnnotationStyleLayout(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,61 +37,8 @@ private func scrollCaptureViewportPointCandidates(
return [point, flippedPoint]
}

private struct NativeScrollCaptureSampleFrame: Sendable {
let region: RGBARegionSnapshot
let source: String
let frameSequence: UInt64
let frameAgeMicroseconds: UInt64
}

private struct NativeScrollCaptureFallbackRequest: Sendable {
let rect: CGRect
let source: CaptureSessionController.FrozenCaptureJobSource
let frameSequence: UInt64
}

private struct NativeScrollCaptureObservation: Sendable {
let sampledFrame: NativeScrollCaptureSampleFrame
let registrationStrategy: String
let result: ScrollObserveResult?
let errorDescription: String?
}

private let nativeScrollCaptureMinimumNonzeroWheelMotionHintRows = 12.0

private struct NativeScrollCapturePreviewUpdate: @unchecked Sendable {
let image: CGImage
let exportWidth: Int
let exportHeight: Int
let result: ScrollObserveResult
let viewportTopYPixels: Int
let viewportHeightPixels: Int
}

private struct NativeScrollCaptureObservationBatch: Sendable {
let observations: [NativeScrollCaptureObservation]
let preview: NativeScrollCapturePreviewUpdate?
let previewErrorDescription: String?
let previewExportMilliseconds: Double?
}

private func writeNativeScrollCaptureDebugDump(_ snapshot: RGBARegionSnapshot, name: String) {
guard
let rawDirectory = ProcessInfo.processInfo.environment["RSNAP_SCROLL_CAPTURE_DUMP_DIR"],
rawDirectory.isEmpty == false,
let pngData = try? RsnapExportEncoder.pngData(from: snapshot)
else {
return
}
let directory = URL(fileURLWithPath: rawDirectory, isDirectory: true)
try? FileManager.default.createDirectory(
at: directory,
withIntermediateDirectories: true
)
let safeName = name.replacingOccurrences(of: "/", with: "_")
try? pngData.write(to: directory.appendingPathComponent("\(safeName).png"))
}

extension CaptureSessionController {
private static let scrollCaptureForwardedEventMarker: Int64 = 0x5253_4E41_5053_4352
private static let scrollCapturePreciseWheelDeltaLimit = 72.0
Expand Down Expand Up @@ -745,7 +692,7 @@ extension CaptureSessionController {
frameAgeMicroseconds: 0
))
}
let batch = Self.nativeScrollCaptureObservationBatch(
let batch = NativeScrollCaptureObservationPipeline.makeBatch(
sampledFrames: sampledFrames,
stitcher: stitcher,
motionRowsHint: motionRowsHint,
Expand All @@ -763,114 +710,6 @@ extension CaptureSessionController {
}
}

nonisolated private static func nativeScrollCaptureObservationBatch(
sampledFrames: [NativeScrollCaptureSampleFrame],
stitcher: RsnapScrollCaptureSession,
motionRowsHint: Int?,
previewRefreshDue: Bool
) -> NativeScrollCaptureObservationBatch {
var observations: [NativeScrollCaptureObservation] = []
var latestPreviewCandidate: NativeScrollCaptureObservation?
for sampledFrame in sampledFrames {
let observation = nativeScrollCaptureObservation(
sampledFrame,
stitcher: stitcher,
motionRowsHint: motionRowsHint
)
if observation.result?.outcome != .noChange {
latestPreviewCandidate = observation
}
observations.append(observation)
}
let preview = nativeScrollCapturePreviewUpdate(
stitcher: stitcher,
candidate: latestPreviewCandidate,
previewRefreshDue: previewRefreshDue
)
return NativeScrollCaptureObservationBatch(
observations: observations,
preview: preview.update,
previewErrorDescription: preview.errorDescription,
previewExportMilliseconds: preview.exportMilliseconds
)
}

nonisolated private static func nativeScrollCaptureObservation(
_ sampledFrame: NativeScrollCaptureSampleFrame,
stitcher: RsnapScrollCaptureSession,
motionRowsHint: Int?
) -> NativeScrollCaptureObservation {
let registrationStrategy = "pairwise"
do {
let result = try stitcher.observeDownwardFrame(
sampledFrame.region,
motionRowsHint: motionRowsHint
)
return NativeScrollCaptureObservation(
sampledFrame: sampledFrame,
registrationStrategy: registrationStrategy,
result: result,
errorDescription: nil
)
} catch {
return NativeScrollCaptureObservation(
sampledFrame: sampledFrame,
registrationStrategy: registrationStrategy,
result: nil,
errorDescription: String(describing: error)
)
}
}

nonisolated private static func nativeScrollCapturePreviewUpdate(
stitcher: RsnapScrollCaptureSession,
candidate: NativeScrollCaptureObservation?,
previewRefreshDue: Bool
) -> (
update: NativeScrollCapturePreviewUpdate?,
errorDescription: String?,
exportMilliseconds: Double?
) {
guard previewRefreshDue, let candidate, let result = candidate.result else {
return (nil, nil, nil)
}
let previewStartedAt = ProcessInfo.processInfo.systemUptime
do {
if let export = try stitcher.exportImage() {
guard let exportImage = NativeHostImageBridge.cgImage(from: export) else {
return (
nil,
"scroll preview export returned no image",
NativeHostTelemetry.milliseconds(since: previewStartedAt)
)
}
return (
NativeScrollCapturePreviewUpdate(
image: exportImage,
exportWidth: export.width,
exportHeight: export.height,
result: result,
viewportTopYPixels: result.currentViewportTopY,
viewportHeightPixels: candidate.sampledFrame.region.height
),
nil,
NativeHostTelemetry.milliseconds(since: previewStartedAt)
)
}
return (
nil,
"scroll preview export returned no image",
NativeHostTelemetry.milliseconds(since: previewStartedAt)
)
} catch {
return (
nil,
String(describing: error),
NativeHostTelemetry.milliseconds(since: previewStartedAt)
)
}
}

private func finishNativeScrollCaptureObservations(
_ batch: NativeScrollCaptureObservationBatch,
captureID: UInt64,
Expand Down
Loading