From 121f2c6b0b045bf4b33dbd05bd9ffcba0199ac96 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Tue, 28 Jul 2026 20:07:25 -0700 Subject: [PATCH] fix(engines): restart capture on mid-session config change in WhisperKit/AppleSpeech/SpeechAnalyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #224 gave ParakeetStreamingEngine an AVAudioEngineConfigurationChange observer that rebuilds capture onto the same feed stream when the input device disconnects/switches/renegotiates mid-dictation. The other three streaming engines had the same hole: AVAudioEngine stops rendering on the change, the session keeps showing "Listening..." and every word after the glitch is silently lost. Port the Parakeet pattern (selector-based observer to stay @Sendable-warning-free, makeRoutedEngine/startCapture split, restart-then-loud-error, CaptureConfigChangePolicy gates) to: - AppleSpeechEngine: rebuild engine + tap appending to the SAME SFSpeechAudioBufferRecognitionRequest, so the recognizer sees one continuous audio stream. - SpeechAnalyzerStreamingEngine: rebuild engine + tap yielding into the SAME raw-buffer AsyncStream. The analyzer-format converter is now ADAPTIVE (rebuilt when the incoming buffer format changes) — a converter pinned to the start-time mic format would drop every buffer from the replacement device, leaving the transcript dead even after a successful capture restart. - WhisperKitStreamingEngine: the mic belongs to WhisperKit's AudioProcessor, so the rebuild lives on WhisperKitStreamHandle (restartCapture mirrors the internal setupEngine via public API and feeds processBuffer, appending to the same decode buffer the realtime loop polls); the engine arms the observer on the processor's engine object at first state diff and re-arms on the replacement after a restart. Restart failure (e.g. no input device left) surfaces through the session error path via CaptureConfigChangePolicy.restartFailedMessage instead of eating speech silently. Each engine's restart path still needs a live-mic device-change pass (unplug/switch mid-dictation); the policy gates are pinned by CaptureConfigChangePolicyTests. Co-Authored-By: Claude --- OpenWhisp/Services/AppleSpeechEngine.swift | 197 +++++++++++---- .../Services/CaptureConfigChangePolicy.swift | 5 +- .../SpeechAnalyzerStreamingEngine.swift | 239 ++++++++++++++---- OpenWhisp/Services/WhisperKitBridge.swift | 81 ++++++ .../Services/WhisperKitStreamingEngine.swift | 165 ++++++++++-- .../CaptureConfigChangePolicyTests.swift | 3 +- 6 files changed, 570 insertions(+), 120 deletions(-) diff --git a/OpenWhisp/Services/AppleSpeechEngine.swift b/OpenWhisp/Services/AppleSpeechEngine.swift index 3d8c736..eacd96e 100644 --- a/OpenWhisp/Services/AppleSpeechEngine.swift +++ b/OpenWhisp/Services/AppleSpeechEngine.swift @@ -2,7 +2,10 @@ import Foundation import AVFoundation import Speech -final class AppleSpeechEngine: StreamingTranscriptionEngine { +// NSObject so the AVAudioEngineConfigurationChange observer can be +// selector-based (the block API takes a hard `@Sendable` closure whose captures +// here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class AppleSpeechEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -46,6 +49,16 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { /// Speech uses its own `AVAudioEngine` whose input node we can retarget directly. @MainActor private var selectedDeviceID = "" + /// What the mid-session configuration-change restart needs to rebuild capture + /// into the live recognition request (see `armConfigChangeObserver`). Non-nil + /// exactly while the observer is armed; cleared on every teardown path. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let request: SFSpeechAudioBufferRecognitionRequest + let myGeneration: Int + } + func selectDevice(_ deviceID: String) { // Callers are @MainActor (AppState); pinning is a plain main-actor store. MainActor.assumeIsolated { @@ -109,46 +122,7 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { request.contextualStrings = contextualStrings } - let engine = AVAudioEngine() - let input = engine.inputNode - - // Route to the selected input device BEFORE reading the format (the format - // follows the device). A CONNECTED selection whose routing fails is a hard - // error — never silently capture a different mic (that was the bug where - // picking a non-default mic still captured the built-in one). A pinned but - // DISCONNECTED device instead falls back to the system default so the - // session can run (the AirPods-disconnect hang). - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - break // input node already follows the default - case .useDevice(let uid): - // Resolved above; a nil here or a setDeviceID failure is still a hard - // error — don't fall through to the default node. - guard let device = AudioInputRouter.resolve(uid: uid), - AudioInputRouter.apply(device, to: engine) else { - throw AppleSpeechError.unavailable(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - } - case .fallbackToDefault(let uid): - NSLog("[AppleSpeechEngine] pinned mic '%@' disconnected — capturing system default", uid) - } - - let format = input.outputFormat(forBus: 0) - // With no input device the format is 0 Hz / 0 ch and installTap raises - // an ObjC NSException that Swift try/catch can't intercept (app crash). - guard format.sampleRate > 0, format.channelCount > 0 else { - throw AppleSpeechError.unavailable("No audio input device available.") - } - - // The tap closure runs on an audio thread but touches no shared session - // state — it only appends the buffer to the (thread-safe) request and reads - // the buffer to publish a level. Nothing here races with start()/stop(). - input.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in - request.append(buffer) - self?.publishLevel(from: buffer) - } + let (engine, format) = try makeRoutedEngine() let myGeneration = generation recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in @@ -189,18 +163,152 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { } } - audioEngine = engine recognitionRequest = request self.recognizer = recognizer - engine.prepare() - try engine.start() + try startCapture(engine: engine, format: format, request: request, myGeneration: myGeneration) // The tap is installed and the AVAudioEngine is running — capture is // live NOW. This engine's start is fully synchronous, so the signal // fires before runStart returns (unlike the chained engines). onStarted?() } + /// Build a fresh AVAudioEngine routed to the pinned (or default) input and + /// return it with its tap format. Shared by session start and the + /// mid-session configuration-change restart. + /// + /// Routing happens BEFORE reading the format (the format follows the + /// device). A CONNECTED selection whose routing fails is a hard error — + /// never silently capture a different mic (that was the bug where picking a + /// non-default mic still captured the built-in one). A pinned but + /// DISCONNECTED device instead falls back to the system default so the + /// session can run (the AirPods-disconnect hang). + @MainActor + private func makeRoutedEngine() throws -> (AVAudioEngine, AVAudioFormat) { + let engine = AVAudioEngine() + let input = engine.inputNode + + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + break // input node already follows the default + case .useDevice(let uid): + // Resolved above; a nil here or a setDeviceID failure is still a hard + // error — don't fall through to the default node. + guard let device = AudioInputRouter.resolve(uid: uid), + AudioInputRouter.apply(device, to: engine) else { + throw AppleSpeechError.unavailable(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + case .fallbackToDefault(let uid): + NSLog("[AppleSpeechEngine] pinned mic '%@' disconnected — capturing system default", uid) + } + + let format = input.outputFormat(forBus: 0) + // With no input device the format is 0 Hz / 0 ch and installTap raises + // an ObjC NSException that Swift try/catch can't intercept (app crash). + guard format.sampleRate > 0, format.channelCount > 0 else { + throw AppleSpeechError.unavailable("No audio input device available.") + } + return (engine, format) + } + + /// Install the tap on `engine`, start it, and arm the configuration-change + /// observer. Shared by session start and the mid-session restart — both + /// append to the SAME recognition request, so the recognizer sees one + /// continuous audio stream. + @MainActor + private func startCapture( + engine: AVAudioEngine, + format: AVAudioFormat, + request: SFSpeechAudioBufferRecognitionRequest, + myGeneration: Int + ) throws { + // The tap closure runs on an audio thread but touches no shared session + // state — it only appends the buffer to the (thread-safe) request and reads + // the buffer to publish a level. Nothing here races with start()/stop(). + engine.inputNode.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in + request.append(buffer) + self?.publishLevel(from: buffer) + } + audioEngine = engine + engine.prepare() + try engine.start() + armConfigChangeObserver(for: engine, request: request, myGeneration: myGeneration) + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT onto the same recognition request: the recognizer keeps + /// everything already heard and the dictation continues, losing only the + /// glitch itself. Rebuild failure (e.g. no input device left) surfaces on + /// the session error path. Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + request: SFSpeechAudioBufferRecognitionRequest, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(request: request, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the request) + // is non-Sendable — the selector target keeps the compile warning-free + // and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the recognition callback. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the recognition callback: a stop or a newer + // session owns the mic now — touch nothing. (Teardown also clears the + // context.) + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: didStop + ) == .restartCapture else { return } + NSLog("[AppleSpeechEngine] input device configuration changed — restarting capture") + removeConfigChangeObserver() + // The session fence held, so audioEngine is the engine the observer was + // armed for (re-arms replace the observer and the context together). + if let stale = audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + audioEngine = nil + do { + let (fresh, format) = try makeRoutedEngine() + try startCapture( + engine: fresh, format: format, + request: context.request, myGeneration: context.myGeneration) + } catch { + onError?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + func stop(cancel: Bool = false) { // Callers are @MainActor (AppState); the session state below is main-confined. MainActor.assumeIsolated { @@ -212,6 +320,7 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { private func runStop(cancel: Bool) { didStop = true + removeConfigChangeObserver() if let audioEngine { audioEngine.inputNode.removeTap(onBus: 0) audioEngine.stop() diff --git a/OpenWhisp/Services/CaptureConfigChangePolicy.swift b/OpenWhisp/Services/CaptureConfigChangePolicy.swift index 30e301e..828a789 100644 --- a/OpenWhisp/Services/CaptureConfigChangePolicy.swift +++ b/OpenWhisp/Services/CaptureConfigChangePolicy.swift @@ -1,8 +1,9 @@ import Foundation /// Pure decision logic for a mid-session `AVAudioEngineConfigurationChange` on a -/// streaming capture engine (Parakeet/WhisperKit/Apple Speech own their own -/// `AVAudioEngine` taps). +/// streaming capture engine (Parakeet/Apple Speech/SpeechAnalyzer own their own +/// `AVAudioEngine` taps; WhisperKit's tap lives on its `AudioProcessor`, rebuilt +/// through `WhisperKitStreamHandle.restartCapture`). /// /// When the input device is disconnected, switched, or renegotiates its format /// mid-dictation, AVAudioEngine STOPS rendering. Without a reaction the session diff --git a/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift b/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift index 598abbe..96bba4c 100644 --- a/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift +++ b/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift @@ -20,7 +20,10 @@ import Speech /// All Speech-framework calls are behind `if #available(macOS 26, *)`, so the /// type compiles and links on macOS 14/15 — where the engine is hidden and, if /// reached, `start()` throws an unavailability error. -final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { +// NSObject so the AVAudioEngineConfigurationChange observer can be +// selector-based (the block API takes a hard `@Sendable` closure whose captures +// here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class SpeechAnalyzerStreamingEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -44,6 +47,16 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { @MainActor private var finalDelivered = false @MainActor private var generation = 0 + /// What the mid-session configuration-change restart needs to rebuild capture + /// into the live analyzer stream (see `armConfigChangeObserver`). Non-nil + /// exactly while the observer is armed; cleared on every teardown path. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let continuation: AsyncStream.Continuation + let myGeneration: Int + } + func selectDevice(_ deviceID: String) { MainActor.assumeIsolated { selectedDeviceID = deviceID } } @@ -81,32 +94,7 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { throw SpeechAnalyzerBridge.BridgeError.unavailableOS } - let engine = AVAudioEngine() - let input = engine.inputNode - - // Route to the pinned input device BEFORE reading the format (same policy - // as AppleSpeechEngine — a disconnected pinned device falls back to the - // system default; a connected one that fails to route is a hard error). - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - break - case .useDevice(let uid): - guard let device = AudioInputRouter.resolve(uid: uid), - AudioInputRouter.apply(device, to: engine) else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( - AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - } - case .fallbackToDefault(let uid): - NSLog("[SpeechAnalyzerStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) - } - - let micFormat = input.outputFormat(forBus: 0) - guard micFormat.sampleRate > 0, micFormat.channelCount > 0 else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale("no audio input device") - } + let (engine, micFormat) = try makeRoutedEngine() // RAW mic-buffer stream: the tap pushes untouched mic-format buffers; the // recognition task converts each to the ANALYZER'S required format before @@ -118,11 +106,6 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { // internally.) let (rawStream, continuation) = AsyncStream.makeStream(of: AVAudioPCMBuffer.self) - input.installTap(onBus: 0, bufferSize: 4096, format: micFormat) { [weak self] buffer, _ in - continuation.yield(buffer) - self?.publishLevel(from: buffer) - } - // Drive the analyzer on a detached task; hop to the main actor to publish // partial/final under the generation fence. recognitionTask = Task { [weak self] in @@ -146,19 +129,23 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { else { throw SpeechAnalyzerBridge.BridgeError.noResult } - let converter: AVAudioConverter? - if analyzerFormat == micFormat { - converter = nil // already compatible; feed as-is - } else { - guard let c = AVAudioConverter(from: micFormat, to: analyzerFormat) else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( - "mic format \(micFormat) can't convert to the analyzer format") - } - converter = c + // Fail fast when the start-time mic format can't reach the + // analyzer format at all — feeding unconverted audio would trap + // the framework (see the raw-stream comment above). + if analyzerFormat != micFormat, + AVAudioConverter(from: micFormat, to: analyzerFormat) == nil { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( + "mic format \(micFormat) can't convert to the analyzer format") } + // Per-buffer ADAPTIVE conversion, not one converter pinned to + // `micFormat`: after a mid-session capture restart (see + // `armConfigChangeObserver`) the replacement device's format + // usually differs, and a converter pinned to the start-time + // format would reject — drop — every buffer from the new device: + // capture restarted, transcript still silently dead. + let converter = AdaptiveFormatConverter(analyzerFormat: analyzerFormat) let inputStream = rawStream.compactMap { buffer -> AnalyzerInput? in - guard let converter else { return AnalyzerInput(buffer: buffer) } - guard let converted = Self.convert(buffer, with: converter, to: analyzerFormat) + guard let converted = converter.convert(buffer) else { return nil } // drop an unconvertible buffer, never trap return AnalyzerInput(buffer: converted) } @@ -197,9 +184,9 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { } } - audioEngine = engine - engine.prepare() - try engine.start() + try startCapture( + engine: engine, format: micFormat, + continuation: continuation, myGeneration: myGeneration) // Model load can lag the tap install; onStarted signals genuine capture. // The tap is installed and the engine is running, so fire it now. onStarted?() @@ -208,10 +195,166 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { #endif } - #if compiler(>=6.2) + /// Build a fresh AVAudioEngine routed to the pinned (or default) input and + /// return it with its tap format. Shared by session start and the + /// mid-session configuration-change restart. Same policy as + /// AppleSpeechEngine: a disconnected pinned device falls back to the system + /// default; a connected one that fails to route is a hard error. + @MainActor + private func makeRoutedEngine() throws -> (AVAudioEngine, AVAudioFormat) { + let engine = AVAudioEngine() + let input = engine.inputNode + + // Route BEFORE reading the format (the format follows the device). + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + break + case .useDevice(let uid): + guard let device = AudioInputRouter.resolve(uid: uid), + AudioInputRouter.apply(device, to: engine) else { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( + AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + case .fallbackToDefault(let uid): + NSLog("[SpeechAnalyzerStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) + } + + let micFormat = input.outputFormat(forBus: 0) + guard micFormat.sampleRate > 0, micFormat.channelCount > 0 else { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale("no audio input device") + } + return (engine, micFormat) + } + + /// Install the tap on `engine`, start it, and arm the configuration-change + /// observer. Shared by session start and the mid-session restart — both + /// yield into the SAME raw-buffer stream, so the analyzer sees one + /// continuous sample sequence (the adaptive converter absorbs a format + /// change across the restart). + @MainActor + private func startCapture( + engine: AVAudioEngine, + format: AVAudioFormat, + continuation: AsyncStream.Continuation, + myGeneration: Int + ) throws { + engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in + continuation.yield(buffer) + self?.publishLevel(from: buffer) + } + audioEngine = engine + engine.prepare() + try engine.start() + armConfigChangeObserver(for: engine, continuation: continuation, myGeneration: myGeneration) + } + + /// Converts tap buffers to the analyzer's required format, REBUILDING the + /// converter whenever the incoming format changes (a mid-session capture + /// restart lands on a device with a different native format). + /// + /// `@unchecked Sendable` because the `compactMap` transform that owns it is + /// `@Sendable`: the mutable converter is touched only from the recognition + /// task's SEQUENTIAL stream consumption (one buffer at a time, in order), + /// so there is no concurrent access to lock against. + private final class AdaptiveFormatConverter: @unchecked Sendable { + private let analyzerFormat: AVAudioFormat + private var converter: AVAudioConverter? + + init(analyzerFormat: AVAudioFormat) { + self.analyzerFormat = analyzerFormat + } + + /// nil when the buffer can't be represented in the analyzer format + /// (converter build or conversion failure) — the caller drops it. + func convert(_ buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + if buffer.format == analyzerFormat { return buffer } // feed as-is + if converter == nil || converter?.inputFormat != buffer.format { + converter = AVAudioConverter(from: buffer.format, to: analyzerFormat) + } + guard let converter else { return nil } + return SpeechAnalyzerStreamingEngine.convert(buffer, with: converter, to: analyzerFormat) + } + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT onto the same raw-buffer stream: the analyzer keeps everything + /// already captured and the dictation continues, losing only the glitch + /// itself. Rebuild failure (e.g. no input device left) surfaces on the + /// session error path. Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + continuation: AsyncStream.Continuation, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(continuation: continuation, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the + // continuation) is non-Sendable — the selector target keeps the compile + // warning-free and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the result callbacks. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the result hops: a stop or a newer session owns + // the mic now — touch nothing. This engine has no didStop flag; + // `finalDelivered` is its "session over" marker (a cancel bumps the + // generation instead). Teardown also clears the context. + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: finalDelivered + ) == .restartCapture else { return } + NSLog("[SpeechAnalyzerStreamingEngine] input device configuration changed — restarting capture") + removeConfigChangeObserver() + // The session fence held, so audioEngine is the engine the observer was + // armed for (re-arms replace the observer and the context together). + if let stale = audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + audioEngine = nil + do { + let (fresh, format) = try makeRoutedEngine() + try startCapture( + engine: fresh, format: format, + continuation: context.continuation, myGeneration: context.myGeneration) + } catch { + onError?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + /// Convert one mic-format buffer to the analyzer's required format. Returns /// nil (caller drops the buffer) on any conversion failure — a dropped chunk /// degrades the transcript; an unconverted chunk traps the Speech framework. + /// (Plain AVFoundation, so it compiles on every toolchain — no compiler gate.) private static func convert( _ buffer: AVAudioPCMBuffer, with converter: AVAudioConverter, to format: AVAudioFormat ) -> AVAudioPCMBuffer? { @@ -233,7 +376,6 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { guard conversionError == nil, out.frameLength > 0 else { return nil } return out } - #endif /// Append a finalized segment to the running transcript and return the whole. @MainActor @@ -262,6 +404,7 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { @MainActor private func runStop(cancel: Bool) { + removeConfigChangeObserver() if let audioEngine { audioEngine.inputNode.removeTap(onBus: 0) audioEngine.stop() diff --git a/OpenWhisp/Services/WhisperKitBridge.swift b/OpenWhisp/Services/WhisperKitBridge.swift index f89ac32..9623ed5 100644 --- a/OpenWhisp/Services/WhisperKitBridge.swift +++ b/OpenWhisp/Services/WhisperKitBridge.swift @@ -1,5 +1,6 @@ import Foundation import CoreAudio // AudioDeviceID (the input-device id threaded to AudioStreamTranscriber) +import AVFoundation // AVAudioEngine (mid-session capture restart on the stream handle) /// Maps OpenWhisp's engine-facing language setting to WhisperKit decoding /// options. Mirrors `WhisperTask` (used for whisper.cpp): the shared @@ -463,6 +464,86 @@ final class WhisperKitStreamHandle { (latest?.fullText ?? "").trimmingCharacters(in: .whitespacesAndNewlines) } + // MARK: Mid-session capture restart (AVAudioEngineConfigurationChange) + + /// The `AVAudioEngine` WhisperKit's `AudioProcessor` is currently capturing + /// with — nil before capture starts, after stop, or on a custom (non-default) + /// audio processor. Exposed so the engine can arm its + /// `AVAudioEngineConfigurationChange` observer on the exact engine object. + var captureAudioEngine: AVAudioEngine? { + (kit.audioProcessor as? AudioProcessor)?.audioEngine + } + + private struct CaptureRestartError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// Tear down the (dead) capture engine and rebuild a fresh one feeding the + /// SAME decode buffer. After an input-device disconnect/switch/format + /// renegotiation AVAudioEngine stops rendering; the realtime decode loop + /// keeps polling `audioSamples` regardless, so appending fresh capture there + /// resumes the transcript with only the glitch itself lost. + /// + /// Mirrors `AudioProcessor.setupEngine` (internal to WhisperKit) with its + /// public pieces: tap in the node's native format, resample to WhisperKit's + /// 16 kHz mono, honor input suppression, and hand the samples to + /// `processBuffer` — which also keeps `audioEnergy` (levels/VAD) and the + /// buffer callback flowing. + func restartCapture(inputDeviceID: AudioDeviceHandle?) throws { + guard let processor = kit.audioProcessor as? AudioProcessor else { + throw CaptureRestartError( + message: "capture restart requires WhisperKit's default audio processor") + } + if let stale = processor.audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + processor.audioEngine = nil + + let engine = AVAudioEngine() + let input = engine.inputNode + // Route BEFORE reading the format (the format follows the device). The + // caller resolved the device under AudioInputRoutingPolicy, so a routing + // failure here is a hard error — never silently capture a different mic. + if let deviceID = inputDeviceID { + try input.auAudioUnit.setDeviceID(deviceID) + } + let format = input.outputFormat(forBus: 0) + // 0 Hz / 0 ch (no input device) would make installTap raise an ObjC + // NSException that Swift can't catch — guard it out. + guard format.sampleRate > 0, format.channelCount > 0 else { + throw CaptureRestartError(message: "No audio input device available.") + } + guard let desiredFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: Double(WhisperKit.sampleRate), + channels: 1, + interleaved: false + ), let converter = AVAudioConverter(from: format, to: desiredFormat) else { + throw CaptureRestartError( + message: "mic format \(format) can't convert to WhisperKit's 16 kHz mono") + } + let bufferSize = AVAudioFrameCount(processor.minBufferLength) + input.installTap(onBus: 0, bufferSize: bufferSize, format: format) { [weak processor] buffer, _ in + guard let processor else { return } + var resampled = buffer + if !buffer.format.sampleRate.isEqual(to: Double(WhisperKit.sampleRate)) { + guard let converted = try? AudioProcessor.resampleBuffer(buffer, with: converter) + else { return } // drop an unconvertible buffer + resampled = converted + } + var samples = AudioProcessor.convertBufferToArray(buffer: resampled) + if processor.isInputSuppressed { + samples = [Float](repeating: 0, count: samples.count) + } + processor.processBuffer(samples) + } + engine.prepare() + try engine.start() + processor.audioEngine = engine + } + /// Ignore stop-time tails shorter than this (0.25 s @ 16 kHz): releasing the /// hotkey right at a decode boundary leaves a sliver of silence that isn't /// worth a decode pass. diff --git a/OpenWhisp/Services/WhisperKitStreamingEngine.swift b/OpenWhisp/Services/WhisperKitStreamingEngine.swift index 7c0f3e1..42e25bf 100644 --- a/OpenWhisp/Services/WhisperKitStreamingEngine.swift +++ b/OpenWhisp/Services/WhisperKitStreamingEngine.swift @@ -1,5 +1,6 @@ import Foundation -import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) +import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) +import AVFoundation // AVAudioEngine (config-change observer on WhisperKit's capture engine) /// Experimental real-time WhisperKit engine (streaming partials). /// @@ -13,7 +14,11 @@ import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) /// /// **Build:** real implementation only under `#if WHISPERKIT`; otherwise a stub /// that reports unavailability, so the default build is unaffected. -final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { +/// +/// NSObject so the AVAudioEngineConfigurationChange observer can be +/// selector-based (the block API takes a hard `@Sendable` closure whose +/// captures here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class WhisperKitStreamingEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -36,6 +41,7 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { init(modelName: String = "openai_whisper-small") { self.modelName = modelName + super.init() } #if WHISPERKIT @@ -74,6 +80,19 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { /// re-press). See `SerialTaskChain`. @MainActor private let lifecycle = SerialTaskChain() + /// What the mid-session configuration-change restart needs (see + /// `armConfigChangeObserver`). Non-nil exactly while the observer is armed; + /// cleared on every teardown path. Unlike the engines that own their tap, + /// the capture rebuild itself lives on the stream handle + /// (`WhisperKitStreamHandle.restartCapture`) because the mic belongs to + /// WhisperKit's `AudioProcessor`. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let callbacks: SessionCallbacks + let myGeneration: Int + } + func start(language: String, prompt: String) throws { let task = WhisperKitTaskMapper.map(languageSetting: language) // Enqueue SYNCHRONOUSLY on the main actor so call order == enqueue order @@ -122,29 +141,13 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { // Resolve the selected input device. It's passed straight into // AudioStreamTranscriber(inputDeviceID:) (our WhisperKit fork backports // upstream #503), which forwards it to startRecordingLive — WhisperKit - // captures the chosen device directly, no system-default swap needed. A - // pinned but DISCONNECTED device falls back to the system default so the - // session can run (the AirPods-disconnect hang); a CONNECTED device that - // fails to resolve at application time stays a hard error. + // captures the chosen device directly, no system-default swap needed. let inputDeviceID: AudioDeviceID? - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - inputDeviceID = nil - case .useDevice(let uid): - // Resolve to the concrete device id. A nil here (device vanished - // between canResolve and now) is treated as unresolved — a hard error, - // NOT a silent nil that would fall back to the system default. - guard let id = AudioInputRouter.resolve(uid: uid)?.deviceID else { - callbacks.error?(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - return - } - inputDeviceID = id - case .fallbackToDefault(let uid): - NSLog("[WhisperKitStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) - inputDeviceID = nil + do { + inputDeviceID = try resolveInputDeviceID() + } catch let setupError as CaptureSetupError { + callbacks.error?(setupError.message) + return } let kit = try await ensureLoaded() @@ -178,6 +181,9 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { } catch { NSLog("[WhisperKitStream] stream error: %@", error.localizedDescription) guard let self, self.transcriber === handle, !self.didFinish else { return } + // The stream is dead — a config-change restart would feed a + // torn-down decode loop, so disarm before erroring out. + self.removeConfigChangeObserver() callbacks.error?("WhisperKit streaming failed: \(error.localizedDescription)") } } @@ -211,6 +217,7 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { /// before any queued start proceeds. @MainActor private func runStop(cancel: Bool, final deliverFinal: ((String) -> Void)?) async { + removeConfigChangeObserver() let handle = transcriber transcriber = nil didFinish = true @@ -237,10 +244,16 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { private func handleState(_ state: WhisperKitStreamState, callbacks: SessionCallbacks) { // First state diff for this stream = capture is live (tap installed, // buffers flowing). Fires before any partial from the same diff so the - // session leaves arming before text starts arriving. + // session leaves arming before text starts arriving. This is also the + // earliest moment WhisperKit's capture AVAudioEngine exists, so the + // configuration-change observer arms here (the caller's generation + // guard makes self.generation this stream's own). if !startedNotified { startedNotified = true callbacks.started?() + if let engine = transcriber?.captureAudioEngine { + armConfigChangeObserver(for: engine, callbacks: callbacks, myGeneration: generation) + } } if let level = state.peakEnergy { // vadLevel is the absolute-curve reading; the display level's @@ -260,6 +273,108 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { callbacks.partial?(text) } + /// Setup failure with a user-facing message (routing refused, device + /// vanished). `LocalizedError` so a generic `localizedDescription` still + /// reads as the message. + private struct CaptureSetupError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// Resolve `selectedDeviceID` to a concrete CoreAudio device id (nil = + /// system default). Shared by session start and the mid-session + /// configuration-change restart. A pinned but DISCONNECTED device falls + /// back to the system default so the session can run (the + /// AirPods-disconnect hang); a CONNECTED device that fails to resolve at + /// application time stays a hard error — NOT a silent nil that would fall + /// back to the system default. + @MainActor + private func resolveInputDeviceID() throws -> AudioDeviceID? { + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + return nil + case .useDevice(let uid): + guard let id = AudioInputRouter.resolve(uid: uid)?.deviceID else { + throw CaptureSetupError(message: AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + return id + case .fallbackToDefault(let uid): + NSLog("[WhisperKitStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) + return nil + } + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT into the same decode buffer (`WhisperKitStreamHandle. + /// restartCapture`): the realtime loop keeps everything already captured + /// and the dictation continues, losing only the glitch itself. Rebuild + /// failure (e.g. no input device left) surfaces on the session error path. + /// Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + callbacks: SessionCallbacks, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(callbacks: callbacks, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the session + // callbacks) is non-Sendable — the selector target keeps the compile + // warning-free and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the state callback. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the state callback: a stop or a newer session + // owns the mic now — touch nothing. (Teardown also clears the context.) + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: didFinish + ) == .restartCapture, let handle = transcriber else { return } + NSLog("[WhisperKitStream] input device configuration changed — restarting capture") + removeConfigChangeObserver() + do { + let deviceID = try resolveInputDeviceID() + try handle.restartCapture(inputDeviceID: deviceID) + // Re-arm on the REPLACEMENT engine — the old observer died with the + // stale engine object. + if let engine = handle.captureAudioEngine { + armConfigChangeObserver( + for: engine, callbacks: context.callbacks, myGeneration: context.myGeneration) + } + } catch { + context.callbacks.error?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + @discardableResult private func ensureLoaded() async throws -> WhisperKitHandle { if let kit = loadedKit { return kit } diff --git a/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift b/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift index 0bd5c21..0a77a00 100644 --- a/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift +++ b/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift @@ -2,7 +2,8 @@ import XCTest @testable import OpenWhispCore /// Pins the gates for the mid-session `AVAudioEngineConfigurationChange` -/// recovery (Parakeet streaming): a live session restarts capture; a stale or +/// recovery shared by ALL streaming capture engines (Parakeet, WhisperKit, +/// Apple Speech, SpeechAnalyzer): a live session restarts capture; a stale or /// stopped session ignores the notification. Without the restart, a device /// disconnect/format renegotiation leaves the session showing "Listening…" /// while capturing NOTHING — words are silently lost for the session's