Fix HUD scrim overlap masking

native/macos-host/Sources/RsnapNativeHostKit/LiveOverlayRenderer.swift

@@ -1258,6 +1258,7 @@ private final class SelectionFlowBandLayer: CALayer {
 }
 
 private final class LiveScrimLayer: CAShapeLayer {
+	private let exclusionMaskLayer = CAShapeLayer()
 	private var renderedBounds = CGRect.null
 	private var focusRect = CGRect.null
 	private var roundedExclusions: [OverlayMaskGeometry.RoundedExclusion] = []
@@ -1286,6 +1287,9 @@ private final class LiveScrimLayer: CAShapeLayer {
 		fillColor = scrimColor
 		strokeColor = nil
 		needsDisplayOnBoundsChange = false
+		exclusionMaskLayer.fillRule = .evenOdd
+		exclusionMaskLayer.fillColor = NSColor.black.cgColor
+		exclusionMaskLayer.strokeColor = nil
 	}
 
 	@available(*, unavailable)
@@ -1314,9 +1318,26 @@ private final class LiveScrimLayer: CAShapeLayer {
 		fillColor = color
 		path = OverlayMaskGeometry.scrimPath(
 			bounds: currentBounds,
-			focusRect: focusRect,
+			focusRect: focusRect
+		)
+		updateExclusionMask(bounds: currentBounds, roundedExclusions: roundedExclusions)
+	}
+
+	private func updateExclusionMask(
+		bounds: CGRect,
+		roundedExclusions: [OverlayMaskGeometry.RoundedExclusion]
+	) {
+		guard !roundedExclusions.isEmpty else {
+			mask = nil
+			return
+		}
+		exclusionMaskLayer.frame = bounds
+		exclusionMaskLayer.contentsScale = contentsScale
+		exclusionMaskLayer.path = OverlayMaskGeometry.evenOddMaskPath(
+			bounds: bounds,
 			roundedExclusions: roundedExclusions
 		)
+		mask = exclusionMaskLayer
 	}
 }
 

native/macos-host/Sources/RsnapNativeHostKit/OverlayMaskGeometry.swift

@@ -29,12 +29,18 @@ package enum OverlayMaskGeometry {
 		context.addPath(
 			scrimPath(
 				bounds: bounds,
-				focusRect: focusRect,
-				roundedExclusions: roundedExclusions,
-				pathExclusions: pathExclusions
+				focusRect: focusRect
 			)
 		)
 		context.fillPath(using: .evenOdd)
+		context.setBlendMode(.clear)
+		for exclusion in roundedExclusions {
+			clearRoundedRect(exclusion, in: context)
+		}
+		for path in pathExclusions {
+			context.addPath(path)
+			context.fillPath()
+		}
 		context.restoreGState()
 	}
 
@@ -90,6 +96,17 @@ package enum OverlayMaskGeometry {
 			transform: nil
 		)
 	}
+
+	private static func clearRoundedRect(
+		_ exclusion: RoundedExclusion,
+		in context: CGContext
+	) {
+		guard let path = roundedPath(for: exclusion) else {
+			return
+		}
+		context.addPath(path)
+		context.fillPath()
+	}
 }
 
 extension CGRect {

native/macos-host/Sources/RsnapNativeHostKitProbe/main.swift

@@ -41,6 +41,7 @@ enum RsnapNativeHostKitProbe {
 			imageSize: imageSize
 		)
 		assertScrimRoundedExclusionKeepsCornersMasked()
+		assertScrimOverlappingRoundedExclusionStaysClear()
 		assertRoundedExclusionMaskKeepsCornersFilled()
 		let minimapExportSize = CGSize(width: 100, height: 200)
 		guard
@@ -224,6 +225,58 @@ enum RsnapNativeHostKitProbe {
 		}
 	}
 
+	private static func assertScrimOverlappingRoundedExclusionStaysClear() {
+		let width = 96
+		let height = 80
+		let byteCount = width * height * 4
+		let data = UnsafeMutablePointer<UInt8>.allocate(capacity: byteCount)
+		data.initialize(repeating: 0, count: byteCount)
+		defer {
+			data.deinitialize(count: byteCount)
+			data.deallocate()
+		}
+		guard
+			let colorSpace = CGColorSpace(name: CGColorSpace.sRGB),
+			let context = CGContext(
+				data: data,
+				width: width,
+				height: height,
+				bitsPerComponent: 8,
+				bytesPerRow: width * 4,
+				space: colorSpace,
+				bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+			)
+		else {
+			fatalError("could not create overlapping scrim geometry probe context")
+		}
+
+		OverlayMaskGeometry.drawScrim(
+			in: context,
+			bounds: CGRect(x: 0, y: 0, width: width, height: height),
+			focusRect: CGRect(x: 42, y: 18, width: 28, height: 28),
+			color: CGColor(red: 0, green: 0, blue: 0, alpha: 1),
+			roundedExclusions: [
+				OverlayMaskGeometry.RoundedExclusion(
+					rect: CGRect(x: 24, y: 18, width: 40, height: 24),
+					cornerRadius: 12
+				)
+			]
+		)
+
+		guard clearPixel(in: data, width: width, height: height, x: 36, yFromBottom: 30)
+		else {
+			fatalError("rounded scrim exclusion did not clear the HUD body outside focus")
+		}
+		guard clearPixel(in: data, width: width, height: height, x: 52, yFromBottom: 30)
+		else {
+			fatalError("overlapping focus and HUD exclusions refilled the scrim")
+		}
+		guard opaquePixel(in: data, width: width, height: height, x: 12, yFromBottom: 12)
+		else {
+			fatalError("overlapping scrim probe did not leave ordinary scrim opaque")
+		}
+	}
+
 	private static func assertRoundedExclusionMaskKeepsCornersFilled() {
 		let width = 80
 		let height = 80