Add live HUD window telemetry

native/macos-host/Sources/RsnapNativeHostKit/CaptureOverlayController.swift

@@ -104,9 +104,11 @@ final class CaptureOverlayController {
 		for window in windows {
 			window.displayIfNeeded()
 		}
-		windowSnapshotFeed.start(
-			desktopFrame: Self.desktopFrame, initialSnapshots: initialWindowSnapshots)
 		let captureID = controller?.activeTelemetryCaptureID ?? 0
+		windowSnapshotFeed.start(
+			desktopFrame: Self.desktopFrame,
+			initialSnapshots: initialWindowSnapshots,
+			captureID: captureID)
 		chromeSampleFeed.start(
 			targetFramesPerSecond: NativeHostDisplayRefresh.samplingFramesPerSecond(),
 			captureID: captureID)
@@ -179,7 +181,7 @@ final class CaptureOverlayController {
 		NativeHostTelemetry.captureEvent(
 			"capture.stream_prepare_started",
 			captureID: controller?.activeTelemetryCaptureID ?? 0,
-			detail: "trigger=\(trigger)"
+			detail: "trigger=\(trigger) overlayWindowCount=\(selfCaptureExceptionWindowIDs.count)"
 		)
 		prepareCaptureStreams()
 	}

native/macos-host/Sources/RsnapNativeHostKit/CaptureSessionController+Live.swift

@@ -161,7 +161,10 @@ extension CaptureSessionController {
 		// include through the app-level exclusion. Overlay windows must stay out
 		// of this list so color sampling sees the desktop under the capture UI.
 		cancelPendingScreenCaptureStreamRelease(reason: "start_capture")
-		liveFrameStream.updateSelfCaptureExceptionWindowIDs(capturableOwnWindowIDs)
+		liveFrameStream.updateSelfCaptureExceptionWindowIDs(
+			capturableOwnWindowIDs,
+			captureID: captureID
+		)
 		let warmStartedAt = ProcessInfo.processInfo.systemUptime
 		let initialSample = warmLiveSamplingIfPossible(
 			at: startPoint,
@@ -179,9 +182,22 @@ extension CaptureSessionController {
 			captureID: captureID
 		)
 		let windowSnapshotStartedAt = ProcessInfo.processInfo.systemUptime
-		let initialWindowSnapshots = WindowSnapshotFeed.snapshots(desktopFrame: desktopFrame)
+		let initialWindowReport = WindowSnapshotFeed.snapshotReport(desktopFrame: desktopFrame)
+		let initialWindowSnapshots = initialWindowReport.snapshots
 		let windowSnapshotMilliseconds =
 			NativeHostTelemetry.milliseconds(since: windowSnapshotStartedAt)
+		NativeHostTelemetry.liveChromeWindowSnapshotRefresh(
+			captureID: captureID,
+			source: "start_capture",
+			totalMilliseconds: windowSnapshotMilliseconds,
+			candidateWindowCount: initialWindowReport.candidateWindowCount,
+			targetableWindowCount: initialWindowReport.snapshots.count,
+			ownWindowCount: initialWindowReport.ownWindowCount,
+			ownTargetableWindowCount: initialWindowReport.ownTargetableWindowCount,
+			highLayerWindowCount: initialWindowReport.highLayerWindowCount,
+			tinyWindowCount: initialWindowReport.tinyWindowCount,
+			transparentWindowCount: initialWindowReport.transparentWindowCount
+		)
 		let initialHighlightedWindow = WindowSnapshotFeed.window(
 			at: startPoint, in: initialWindowSnapshots)
 		chromeState.rgbSample = initialRgbSample
@@ -284,6 +300,12 @@ extension CaptureSessionController {
 				detail: "start_capture_complete_stream"
 			)
 		} else {
+			NativeHostTelemetry.captureEvent(
+				"capture.self_capture_rebuild_requested",
+				captureID: captureID,
+				detail:
+					"overlayWindowCount=\(selfCaptureExceptionWindowIDs.count) capturableOwnWindowCount=\(capturableOwnWindowIDs.count)"
+			)
 			_ = warmLiveSamplingIfPossible(
 				at: startPoint,
 				source: "capture_overlay_preflight",

native/macos-host/Sources/RsnapNativeHostKit/LiveFrameStream.swift

@@ -47,8 +47,12 @@ final class LiveFrameStreamBroker: @unchecked Sendable {
 		)
 	}
 
-	func updateSelfCaptureExceptionWindowIDs(_ windowIDs: Set<CGWindowID>) {
+	func updateSelfCaptureExceptionWindowIDs(
+		_ windowIDs: Set<CGWindowID>,
+		captureID: UInt64 = 0
+	) {
 		stateLock.lock()
+		let previousWindowCount = selfCaptureExceptionWindowIDs.count
 		guard windowIDs != selfCaptureExceptionWindowIDs else {
 			stateLock.unlock()
 			return
@@ -65,6 +69,12 @@ final class LiveFrameStreamBroker: @unchecked Sendable {
 			sampler = Self.makeSampler(exceptionWindowIDs: windowIDs)
 		}
 		stateLock.unlock()
+		NativeHostTelemetry.liveStreamSelfCaptureExceptionUpdate(
+			captureID: captureID,
+			previousWindowCount: previousWindowCount,
+			nextWindowCount: windowIDs.count,
+			samplerRebuilt: oldSampler != nil
+		)
 		try? oldSampler?.reset()
 	}
 

native/macos-host/Sources/RsnapNativeHostKit/LiveOverlayRenderer.swift

@@ -42,20 +42,57 @@ private enum LiveOverlayTypography {
 }
 
 final class WindowSnapshotFeed {
+	struct SnapshotReport {
+		let snapshots: [WindowSnapshot]
+		let candidateWindowCount: Int
+		let ownWindowCount: Int
+		let ownTargetableWindowCount: Int
+		let highLayerWindowCount: Int
+		let tinyWindowCount: Int
+		let transparentWindowCount: Int
+	}
+
 	private static let ownPID = ProcessInfo.processInfo.processIdentifier
 	private static let maxWindowLayerForTargeting = 3
+	private static let slowSnapshotRefreshThresholdMilliseconds = 8.0
+	private static let telemetrySummaryInterval: TimeInterval = 1.0
 	private let queue = DispatchQueue(
 		label: "ink.hack.rsnap.native-host.window-snapshot-feed", qos: .userInitiated)
 	private let stateLock = NSLock()
+	private let snapshotRefreshDurationMetric = NativeHostTelemetry.distribution(
+		"live_chrome.window_snapshot_refresh_duration",
+		category: "LiveChromeTelemetry",
+		batchSize: 30
+	)
+	private let snapshotCandidateWindowCountMetric = NativeHostTelemetry.distribution(
+		"live_chrome.window_snapshot_candidate_count",
+		category: "LiveChromeTelemetry",
+		unit: "windows",
+		batchSize: 30
+	)
+	private let snapshotTargetableWindowCountMetric = NativeHostTelemetry.distribution(
+		"live_chrome.window_snapshot_targetable_count",
+		category: "LiveChromeTelemetry",
+		unit: "windows",
+		batchSize: 30
+	)
 	private var timer: DispatchSourceTimer?
 	private var desktopFrame: CGRect = .null
 	private var latestSnapshots: [WindowSnapshot] = []
+	private var captureID: UInt64 = 0
+	private var lastTelemetrySummaryUptime: TimeInterval = 0
 
-	func start(desktopFrame: CGRect, initialSnapshots: [WindowSnapshot] = []) {
+	func start(
+		desktopFrame: CGRect,
+		initialSnapshots: [WindowSnapshot] = [],
+		captureID: UInt64 = 0
+	) {
 		stop()
 		stateLock.lock()
 		self.desktopFrame = desktopFrame
 		latestSnapshots = initialSnapshots
+		self.captureID = captureID
+		lastTelemetrySummaryUptime = 0
 		stateLock.unlock()
 		let timer = DispatchSource.makeTimerSource(queue: queue)
 		timer.schedule(
@@ -72,6 +109,8 @@ final class WindowSnapshotFeed {
 		timer = nil
 		stateLock.lock()
 		latestSnapshots.removeAll()
+		captureID = 0
+		lastTelemetrySummaryUptime = 0
 		stateLock.unlock()
 	}
 
@@ -83,24 +122,38 @@ final class WindowSnapshotFeed {
 	}
 
 	static func snapshots(desktopFrame: CGRect) -> [WindowSnapshot] {
+		snapshotReport(desktopFrame: desktopFrame).snapshots
+	}
+
+	static func snapshotReport(desktopFrame: CGRect) -> SnapshotReport {
 		let candidateWindows =
 			(CGWindowListCopyWindowInfo(
 				[.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
 				as? [[String: Any]])
 			?? []
 		var snapshots: [WindowSnapshot] = []
+		var ownWindowCount = 0
+		var ownTargetableWindowCount = 0
+		var highLayerWindowCount = 0
+		var tinyWindowCount = 0
+		var transparentWindowCount = 0
 		for info in candidateWindows {
 			let isOnScreen = (info[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue ?? false
 			let ownerPID = (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value ?? -1
+			if ownerPID == ownPID {
+				ownWindowCount += 1
+			}
 			if isOnScreen == false {
 				continue
 			}
 			let alpha = (info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1
 			if alpha < 0.05 {
+				transparentWindowCount += 1
 				continue
 			}
 			let layer = (info[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
 			if layer < 0 || layer > maxWindowLayerForTargeting {
+				highLayerWindowCount += 1
 				continue
 			}
 			if ownerPID == ownPID && !Self.isTargetableOwnWindow(info, layer: layer) {
@@ -120,12 +173,24 @@ final class WindowSnapshotFeed {
 				height: quartzBounds.height
 			)
 			if appKitBounds.width < 40 || appKitBounds.height < 40 {
+				tinyWindowCount += 1
 				continue
 			}
 			let windowID = (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value
+			if ownerPID == ownPID {
+				ownTargetableWindowCount += 1
+			}
 			snapshots.append(WindowSnapshot(windowID: windowID, frame: appKitBounds))
 		}
-		return snapshots
+		return SnapshotReport(
+			snapshots: snapshots,
+			candidateWindowCount: candidateWindows.count,
+			ownWindowCount: ownWindowCount,
+			ownTargetableWindowCount: ownTargetableWindowCount,
+			highLayerWindowCount: highLayerWindowCount,
+			tinyWindowCount: tinyWindowCount,
+			transparentWindowCount: transparentWindowCount
+		)
 	}
 
 	private static func isTargetableOwnWindow(_ info: [String: Any], layer: Int) -> Bool {
@@ -141,13 +206,41 @@ final class WindowSnapshotFeed {
 	}
 
 	private func refresh() {
+		let startedAt = ProcessInfo.processInfo.systemUptime
 		stateLock.lock()
 		let desktopFrame = self.desktopFrame
+		let captureID = self.captureID
 		stateLock.unlock()
-		let snapshots = Self.snapshots(desktopFrame: desktopFrame)
+		let report = Self.snapshotReport(desktopFrame: desktopFrame)
+		let totalMilliseconds = NativeHostTelemetry.milliseconds(since: startedAt)
 		stateLock.lock()
-		latestSnapshots = snapshots
+		latestSnapshots = report.snapshots
+		let summaryDue =
+			startedAt - lastTelemetrySummaryUptime >= Self.telemetrySummaryInterval
+		if summaryDue {
+			lastTelemetrySummaryUptime = startedAt
+		}
 		stateLock.unlock()
+		snapshotRefreshDurationMetric.record(totalMilliseconds)
+		snapshotCandidateWindowCountMetric.record(Double(report.candidateWindowCount))
+		snapshotTargetableWindowCountMetric.record(Double(report.snapshots.count))
+		if summaryDue || totalMilliseconds >= Self.slowSnapshotRefreshThresholdMilliseconds {
+			let telemetrySource =
+				totalMilliseconds >= Self.slowSnapshotRefreshThresholdMilliseconds
+				? "periodic_slow" : "periodic_summary"
+			NativeHostTelemetry.liveChromeWindowSnapshotRefresh(
+				captureID: captureID,
+				source: telemetrySource,
+				totalMilliseconds: totalMilliseconds,
+				candidateWindowCount: report.candidateWindowCount,
+				targetableWindowCount: report.snapshots.count,
+				ownWindowCount: report.ownWindowCount,
+				ownTargetableWindowCount: report.ownTargetableWindowCount,
+				highLayerWindowCount: report.highLayerWindowCount,
+				tinyWindowCount: report.tinyWindowCount,
+				transparentWindowCount: report.transparentWindowCount
+			)
+		}
 	}
 }
 

native/macos-host/Sources/RsnapNativeHostKit/NativeHostTelemetry.swift

@@ -183,6 +183,34 @@ enum NativeHostTelemetry {
 		)
 	}
 
+	static func liveChromeWindowSnapshotRefresh(
+		captureID: UInt64,
+		source: String,
+		totalMilliseconds: Double,
+		candidateWindowCount: Int,
+		targetableWindowCount: Int,
+		ownWindowCount: Int,
+		ownTargetableWindowCount: Int,
+		highLayerWindowCount: Int,
+		tinyWindowCount: Int,
+		transparentWindowCount: Int
+	) {
+		liveChromeLogger.info(
+			"schema=\(schema, privacy: .public) runID=\(runID, privacy: .public) captureID=\(captureID, privacy: .public) event=live_chrome.window_snapshot_refresh source=\(source, privacy: .public) totalMs=\(totalMilliseconds, format: .fixed(precision: 2), privacy: .public) candidateWindowCount=\(candidateWindowCount, privacy: .public) targetableWindowCount=\(targetableWindowCount, privacy: .public) ownWindowCount=\(ownWindowCount, privacy: .public) ownTargetableWindowCount=\(ownTargetableWindowCount, privacy: .public) highLayerWindowCount=\(highLayerWindowCount, privacy: .public) tinyWindowCount=\(tinyWindowCount, privacy: .public) transparentWindowCount=\(transparentWindowCount, privacy: .public)"
+		)
+	}
+
+	static func liveStreamSelfCaptureExceptionUpdate(
+		captureID: UInt64,
+		previousWindowCount: Int,
+		nextWindowCount: Int,
+		samplerRebuilt: Bool
+	) {
+		liveChromeLogger.info(
+			"schema=\(schema, privacy: .public) runID=\(runID, privacy: .public) captureID=\(captureID, privacy: .public) event=live_chrome.self_capture_exception_update previousWindowCount=\(previousWindowCount, privacy: .public) nextWindowCount=\(nextWindowCount, privacy: .public) samplerRebuilt=\(samplerRebuilt, privacy: .public)"
+		)
+	}
+
 	static func liveChromeInputSummary(
 		captureID: UInt64,
 		reason: String,

packages/rsnap-overlay/src/frozen_export.rs

@@ -266,7 +266,7 @@ pub fn render_frozen_overlay_export_rgba(
 fn rgba_image_from_bytes(width: u32, height: u32, rgba: &[u8]) -> Result<RgbaImage> {
 	let expected_len = usize::try_from(width)
 		.ok()
-		.and_then(|width| usize::try_from(height).ok().map(|height| (width, height)))
+		.zip(usize::try_from(height).ok())
 		.and_then(|(width, height)| width.checked_mul(height))
 		.and_then(|pixels| pixels.checked_mul(4))
 		.ok_or_else(|| eyre::eyre!("frozen-overlay export dimensions overflow"))?;

packages/rsnap-overlay/src/lib.rs

@@ -172,7 +172,7 @@ pub mod scroll_stitching {
 	fn rgba_image_from_bytes(width: u32, height: u32, rgba: &[u8]) -> Result<RgbaImage> {
 		let expected_len = usize::try_from(width)
 			.ok()
-			.and_then(|width| usize::try_from(height).ok().map(|height| (width, height)))
+			.zip(usize::try_from(height).ok())
 			.and_then(|(width, height)| width.checked_mul(height))
 			.and_then(|pixels| pixels.checked_mul(4))
 			.ok_or_else(|| eyre::eyre!("scroll-capture frame dimensions overflow"))?;