diff --git a/OpenWhisp/Services/AppleSpeechEngine.swift b/OpenWhisp/Services/AppleSpeechEngine.swift index 3d8c736..eacd96e 100644 --- a/OpenWhisp/Services/AppleSpeechEngine.swift +++ b/OpenWhisp/Services/AppleSpeechEngine.swift @@ -2,7 +2,10 @@ import Foundation import AVFoundation import Speech -final class AppleSpeechEngine: StreamingTranscriptionEngine { +// NSObject so the AVAudioEngineConfigurationChange observer can be +// selector-based (the block API takes a hard `@Sendable` closure whose captures +// here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class AppleSpeechEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -46,6 +49,16 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { /// Speech uses its own `AVAudioEngine` whose input node we can retarget directly. @MainActor private var selectedDeviceID = "" + /// What the mid-session configuration-change restart needs to rebuild capture + /// into the live recognition request (see `armConfigChangeObserver`). Non-nil + /// exactly while the observer is armed; cleared on every teardown path. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let request: SFSpeechAudioBufferRecognitionRequest + let myGeneration: Int + } + func selectDevice(_ deviceID: String) { // Callers are @MainActor (AppState); pinning is a plain main-actor store. MainActor.assumeIsolated { @@ -109,46 +122,7 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { request.contextualStrings = contextualStrings } - let engine = AVAudioEngine() - let input = engine.inputNode - - // Route to the selected input device BEFORE reading the format (the format - // follows the device). A CONNECTED selection whose routing fails is a hard - // error — never silently capture a different mic (that was the bug where - // picking a non-default mic still captured the built-in one). A pinned but - // DISCONNECTED device instead falls back to the system default so the - // session can run (the AirPods-disconnect hang). - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - break // input node already follows the default - case .useDevice(let uid): - // Resolved above; a nil here or a setDeviceID failure is still a hard - // error — don't fall through to the default node. - guard let device = AudioInputRouter.resolve(uid: uid), - AudioInputRouter.apply(device, to: engine) else { - throw AppleSpeechError.unavailable(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - } - case .fallbackToDefault(let uid): - NSLog("[AppleSpeechEngine] pinned mic '%@' disconnected — capturing system default", uid) - } - - let format = input.outputFormat(forBus: 0) - // With no input device the format is 0 Hz / 0 ch and installTap raises - // an ObjC NSException that Swift try/catch can't intercept (app crash). - guard format.sampleRate > 0, format.channelCount > 0 else { - throw AppleSpeechError.unavailable("No audio input device available.") - } - - // The tap closure runs on an audio thread but touches no shared session - // state — it only appends the buffer to the (thread-safe) request and reads - // the buffer to publish a level. Nothing here races with start()/stop(). - input.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in - request.append(buffer) - self?.publishLevel(from: buffer) - } + let (engine, format) = try makeRoutedEngine() let myGeneration = generation recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in @@ -189,18 +163,152 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { } } - audioEngine = engine recognitionRequest = request self.recognizer = recognizer - engine.prepare() - try engine.start() + try startCapture(engine: engine, format: format, request: request, myGeneration: myGeneration) // The tap is installed and the AVAudioEngine is running — capture is // live NOW. This engine's start is fully synchronous, so the signal // fires before runStart returns (unlike the chained engines). onStarted?() } + /// Build a fresh AVAudioEngine routed to the pinned (or default) input and + /// return it with its tap format. Shared by session start and the + /// mid-session configuration-change restart. + /// + /// Routing happens BEFORE reading the format (the format follows the + /// device). A CONNECTED selection whose routing fails is a hard error — + /// never silently capture a different mic (that was the bug where picking a + /// non-default mic still captured the built-in one). A pinned but + /// DISCONNECTED device instead falls back to the system default so the + /// session can run (the AirPods-disconnect hang). + @MainActor + private func makeRoutedEngine() throws -> (AVAudioEngine, AVAudioFormat) { + let engine = AVAudioEngine() + let input = engine.inputNode + + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + break // input node already follows the default + case .useDevice(let uid): + // Resolved above; a nil here or a setDeviceID failure is still a hard + // error — don't fall through to the default node. + guard let device = AudioInputRouter.resolve(uid: uid), + AudioInputRouter.apply(device, to: engine) else { + throw AppleSpeechError.unavailable(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + case .fallbackToDefault(let uid): + NSLog("[AppleSpeechEngine] pinned mic '%@' disconnected — capturing system default", uid) + } + + let format = input.outputFormat(forBus: 0) + // With no input device the format is 0 Hz / 0 ch and installTap raises + // an ObjC NSException that Swift try/catch can't intercept (app crash). + guard format.sampleRate > 0, format.channelCount > 0 else { + throw AppleSpeechError.unavailable("No audio input device available.") + } + return (engine, format) + } + + /// Install the tap on `engine`, start it, and arm the configuration-change + /// observer. Shared by session start and the mid-session restart — both + /// append to the SAME recognition request, so the recognizer sees one + /// continuous audio stream. + @MainActor + private func startCapture( + engine: AVAudioEngine, + format: AVAudioFormat, + request: SFSpeechAudioBufferRecognitionRequest, + myGeneration: Int + ) throws { + // The tap closure runs on an audio thread but touches no shared session + // state — it only appends the buffer to the (thread-safe) request and reads + // the buffer to publish a level. Nothing here races with start()/stop(). + engine.inputNode.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in + request.append(buffer) + self?.publishLevel(from: buffer) + } + audioEngine = engine + engine.prepare() + try engine.start() + armConfigChangeObserver(for: engine, request: request, myGeneration: myGeneration) + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT onto the same recognition request: the recognizer keeps + /// everything already heard and the dictation continues, losing only the + /// glitch itself. Rebuild failure (e.g. no input device left) surfaces on + /// the session error path. Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + request: SFSpeechAudioBufferRecognitionRequest, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(request: request, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the request) + // is non-Sendable — the selector target keeps the compile warning-free + // and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the recognition callback. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the recognition callback: a stop or a newer + // session owns the mic now — touch nothing. (Teardown also clears the + // context.) + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: didStop + ) == .restartCapture else { return } + NSLog("[AppleSpeechEngine] input device configuration changed — restarting capture") + removeConfigChangeObserver() + // The session fence held, so audioEngine is the engine the observer was + // armed for (re-arms replace the observer and the context together). + if let stale = audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + audioEngine = nil + do { + let (fresh, format) = try makeRoutedEngine() + try startCapture( + engine: fresh, format: format, + request: context.request, myGeneration: context.myGeneration) + } catch { + onError?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + func stop(cancel: Bool = false) { // Callers are @MainActor (AppState); the session state below is main-confined. MainActor.assumeIsolated { @@ -212,6 +320,7 @@ final class AppleSpeechEngine: StreamingTranscriptionEngine { private func runStop(cancel: Bool) { didStop = true + removeConfigChangeObserver() if let audioEngine { audioEngine.inputNode.removeTap(onBus: 0) audioEngine.stop() diff --git a/OpenWhisp/Services/CaptureConfigChangePolicy.swift b/OpenWhisp/Services/CaptureConfigChangePolicy.swift index 30e301e..828a789 100644 --- a/OpenWhisp/Services/CaptureConfigChangePolicy.swift +++ b/OpenWhisp/Services/CaptureConfigChangePolicy.swift @@ -1,8 +1,9 @@ import Foundation /// Pure decision logic for a mid-session `AVAudioEngineConfigurationChange` on a -/// streaming capture engine (Parakeet/WhisperKit/Apple Speech own their own -/// `AVAudioEngine` taps). +/// streaming capture engine (Parakeet/Apple Speech/SpeechAnalyzer own their own +/// `AVAudioEngine` taps; WhisperKit's tap lives on its `AudioProcessor`, rebuilt +/// through `WhisperKitStreamHandle.restartCapture`). /// /// When the input device is disconnected, switched, or renegotiates its format /// mid-dictation, AVAudioEngine STOPS rendering. Without a reaction the session diff --git a/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift b/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift index 598abbe..96bba4c 100644 --- a/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift +++ b/OpenWhisp/Services/SpeechAnalyzerStreamingEngine.swift @@ -20,7 +20,10 @@ import Speech /// All Speech-framework calls are behind `if #available(macOS 26, *)`, so the /// type compiles and links on macOS 14/15 — where the engine is hidden and, if /// reached, `start()` throws an unavailability error. -final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { +// NSObject so the AVAudioEngineConfigurationChange observer can be +// selector-based (the block API takes a hard `@Sendable` closure whose captures +// here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class SpeechAnalyzerStreamingEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -44,6 +47,16 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { @MainActor private var finalDelivered = false @MainActor private var generation = 0 + /// What the mid-session configuration-change restart needs to rebuild capture + /// into the live analyzer stream (see `armConfigChangeObserver`). Non-nil + /// exactly while the observer is armed; cleared on every teardown path. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let continuation: AsyncStream.Continuation + let myGeneration: Int + } + func selectDevice(_ deviceID: String) { MainActor.assumeIsolated { selectedDeviceID = deviceID } } @@ -81,32 +94,7 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { throw SpeechAnalyzerBridge.BridgeError.unavailableOS } - let engine = AVAudioEngine() - let input = engine.inputNode - - // Route to the pinned input device BEFORE reading the format (same policy - // as AppleSpeechEngine — a disconnected pinned device falls back to the - // system default; a connected one that fails to route is a hard error). - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - break - case .useDevice(let uid): - guard let device = AudioInputRouter.resolve(uid: uid), - AudioInputRouter.apply(device, to: engine) else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( - AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - } - case .fallbackToDefault(let uid): - NSLog("[SpeechAnalyzerStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) - } - - let micFormat = input.outputFormat(forBus: 0) - guard micFormat.sampleRate > 0, micFormat.channelCount > 0 else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale("no audio input device") - } + let (engine, micFormat) = try makeRoutedEngine() // RAW mic-buffer stream: the tap pushes untouched mic-format buffers; the // recognition task converts each to the ANALYZER'S required format before @@ -118,11 +106,6 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { // internally.) let (rawStream, continuation) = AsyncStream.makeStream(of: AVAudioPCMBuffer.self) - input.installTap(onBus: 0, bufferSize: 4096, format: micFormat) { [weak self] buffer, _ in - continuation.yield(buffer) - self?.publishLevel(from: buffer) - } - // Drive the analyzer on a detached task; hop to the main actor to publish // partial/final under the generation fence. recognitionTask = Task { [weak self] in @@ -146,19 +129,23 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { else { throw SpeechAnalyzerBridge.BridgeError.noResult } - let converter: AVAudioConverter? - if analyzerFormat == micFormat { - converter = nil // already compatible; feed as-is - } else { - guard let c = AVAudioConverter(from: micFormat, to: analyzerFormat) else { - throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( - "mic format \(micFormat) can't convert to the analyzer format") - } - converter = c + // Fail fast when the start-time mic format can't reach the + // analyzer format at all — feeding unconverted audio would trap + // the framework (see the raw-stream comment above). + if analyzerFormat != micFormat, + AVAudioConverter(from: micFormat, to: analyzerFormat) == nil { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( + "mic format \(micFormat) can't convert to the analyzer format") } + // Per-buffer ADAPTIVE conversion, not one converter pinned to + // `micFormat`: after a mid-session capture restart (see + // `armConfigChangeObserver`) the replacement device's format + // usually differs, and a converter pinned to the start-time + // format would reject — drop — every buffer from the new device: + // capture restarted, transcript still silently dead. + let converter = AdaptiveFormatConverter(analyzerFormat: analyzerFormat) let inputStream = rawStream.compactMap { buffer -> AnalyzerInput? in - guard let converter else { return AnalyzerInput(buffer: buffer) } - guard let converted = Self.convert(buffer, with: converter, to: analyzerFormat) + guard let converted = converter.convert(buffer) else { return nil } // drop an unconvertible buffer, never trap return AnalyzerInput(buffer: converted) } @@ -197,9 +184,9 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { } } - audioEngine = engine - engine.prepare() - try engine.start() + try startCapture( + engine: engine, format: micFormat, + continuation: continuation, myGeneration: myGeneration) // Model load can lag the tap install; onStarted signals genuine capture. // The tap is installed and the engine is running, so fire it now. onStarted?() @@ -208,10 +195,166 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { #endif } - #if compiler(>=6.2) + /// Build a fresh AVAudioEngine routed to the pinned (or default) input and + /// return it with its tap format. Shared by session start and the + /// mid-session configuration-change restart. Same policy as + /// AppleSpeechEngine: a disconnected pinned device falls back to the system + /// default; a connected one that fails to route is a hard error. + @MainActor + private func makeRoutedEngine() throws -> (AVAudioEngine, AVAudioFormat) { + let engine = AVAudioEngine() + let input = engine.inputNode + + // Route BEFORE reading the format (the format follows the device). + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + break + case .useDevice(let uid): + guard let device = AudioInputRouter.resolve(uid: uid), + AudioInputRouter.apply(device, to: engine) else { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale( + AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + case .fallbackToDefault(let uid): + NSLog("[SpeechAnalyzerStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) + } + + let micFormat = input.outputFormat(forBus: 0) + guard micFormat.sampleRate > 0, micFormat.channelCount > 0 else { + throw SpeechAnalyzerBridge.BridgeError.unsupportedLocale("no audio input device") + } + return (engine, micFormat) + } + + /// Install the tap on `engine`, start it, and arm the configuration-change + /// observer. Shared by session start and the mid-session restart — both + /// yield into the SAME raw-buffer stream, so the analyzer sees one + /// continuous sample sequence (the adaptive converter absorbs a format + /// change across the restart). + @MainActor + private func startCapture( + engine: AVAudioEngine, + format: AVAudioFormat, + continuation: AsyncStream.Continuation, + myGeneration: Int + ) throws { + engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in + continuation.yield(buffer) + self?.publishLevel(from: buffer) + } + audioEngine = engine + engine.prepare() + try engine.start() + armConfigChangeObserver(for: engine, continuation: continuation, myGeneration: myGeneration) + } + + /// Converts tap buffers to the analyzer's required format, REBUILDING the + /// converter whenever the incoming format changes (a mid-session capture + /// restart lands on a device with a different native format). + /// + /// `@unchecked Sendable` because the `compactMap` transform that owns it is + /// `@Sendable`: the mutable converter is touched only from the recognition + /// task's SEQUENTIAL stream consumption (one buffer at a time, in order), + /// so there is no concurrent access to lock against. + private final class AdaptiveFormatConverter: @unchecked Sendable { + private let analyzerFormat: AVAudioFormat + private var converter: AVAudioConverter? + + init(analyzerFormat: AVAudioFormat) { + self.analyzerFormat = analyzerFormat + } + + /// nil when the buffer can't be represented in the analyzer format + /// (converter build or conversion failure) — the caller drops it. + func convert(_ buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + if buffer.format == analyzerFormat { return buffer } // feed as-is + if converter == nil || converter?.inputFormat != buffer.format { + converter = AVAudioConverter(from: buffer.format, to: analyzerFormat) + } + guard let converter else { return nil } + return SpeechAnalyzerStreamingEngine.convert(buffer, with: converter, to: analyzerFormat) + } + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT onto the same raw-buffer stream: the analyzer keeps everything + /// already captured and the dictation continues, losing only the glitch + /// itself. Rebuild failure (e.g. no input device left) surfaces on the + /// session error path. Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + continuation: AsyncStream.Continuation, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(continuation: continuation, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the + // continuation) is non-Sendable — the selector target keeps the compile + // warning-free and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the result callbacks. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the result hops: a stop or a newer session owns + // the mic now — touch nothing. This engine has no didStop flag; + // `finalDelivered` is its "session over" marker (a cancel bumps the + // generation instead). Teardown also clears the context. + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: finalDelivered + ) == .restartCapture else { return } + NSLog("[SpeechAnalyzerStreamingEngine] input device configuration changed — restarting capture") + removeConfigChangeObserver() + // The session fence held, so audioEngine is the engine the observer was + // armed for (re-arms replace the observer and the context together). + if let stale = audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + audioEngine = nil + do { + let (fresh, format) = try makeRoutedEngine() + try startCapture( + engine: fresh, format: format, + continuation: context.continuation, myGeneration: context.myGeneration) + } catch { + onError?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + /// Convert one mic-format buffer to the analyzer's required format. Returns /// nil (caller drops the buffer) on any conversion failure — a dropped chunk /// degrades the transcript; an unconverted chunk traps the Speech framework. + /// (Plain AVFoundation, so it compiles on every toolchain — no compiler gate.) private static func convert( _ buffer: AVAudioPCMBuffer, with converter: AVAudioConverter, to format: AVAudioFormat ) -> AVAudioPCMBuffer? { @@ -233,7 +376,6 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { guard conversionError == nil, out.frameLength > 0 else { return nil } return out } - #endif /// Append a finalized segment to the running transcript and return the whole. @MainActor @@ -262,6 +404,7 @@ final class SpeechAnalyzerStreamingEngine: StreamingTranscriptionEngine { @MainActor private func runStop(cancel: Bool) { + removeConfigChangeObserver() if let audioEngine { audioEngine.inputNode.removeTap(onBus: 0) audioEngine.stop() diff --git a/OpenWhisp/Services/WhisperKitBridge.swift b/OpenWhisp/Services/WhisperKitBridge.swift index f89ac32..9623ed5 100644 --- a/OpenWhisp/Services/WhisperKitBridge.swift +++ b/OpenWhisp/Services/WhisperKitBridge.swift @@ -1,5 +1,6 @@ import Foundation import CoreAudio // AudioDeviceID (the input-device id threaded to AudioStreamTranscriber) +import AVFoundation // AVAudioEngine (mid-session capture restart on the stream handle) /// Maps OpenWhisp's engine-facing language setting to WhisperKit decoding /// options. Mirrors `WhisperTask` (used for whisper.cpp): the shared @@ -463,6 +464,86 @@ final class WhisperKitStreamHandle { (latest?.fullText ?? "").trimmingCharacters(in: .whitespacesAndNewlines) } + // MARK: Mid-session capture restart (AVAudioEngineConfigurationChange) + + /// The `AVAudioEngine` WhisperKit's `AudioProcessor` is currently capturing + /// with — nil before capture starts, after stop, or on a custom (non-default) + /// audio processor. Exposed so the engine can arm its + /// `AVAudioEngineConfigurationChange` observer on the exact engine object. + var captureAudioEngine: AVAudioEngine? { + (kit.audioProcessor as? AudioProcessor)?.audioEngine + } + + private struct CaptureRestartError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// Tear down the (dead) capture engine and rebuild a fresh one feeding the + /// SAME decode buffer. After an input-device disconnect/switch/format + /// renegotiation AVAudioEngine stops rendering; the realtime decode loop + /// keeps polling `audioSamples` regardless, so appending fresh capture there + /// resumes the transcript with only the glitch itself lost. + /// + /// Mirrors `AudioProcessor.setupEngine` (internal to WhisperKit) with its + /// public pieces: tap in the node's native format, resample to WhisperKit's + /// 16 kHz mono, honor input suppression, and hand the samples to + /// `processBuffer` — which also keeps `audioEnergy` (levels/VAD) and the + /// buffer callback flowing. + func restartCapture(inputDeviceID: AudioDeviceHandle?) throws { + guard let processor = kit.audioProcessor as? AudioProcessor else { + throw CaptureRestartError( + message: "capture restart requires WhisperKit's default audio processor") + } + if let stale = processor.audioEngine { + stale.inputNode.removeTap(onBus: 0) + stale.stop() + } + processor.audioEngine = nil + + let engine = AVAudioEngine() + let input = engine.inputNode + // Route BEFORE reading the format (the format follows the device). The + // caller resolved the device under AudioInputRoutingPolicy, so a routing + // failure here is a hard error — never silently capture a different mic. + if let deviceID = inputDeviceID { + try input.auAudioUnit.setDeviceID(deviceID) + } + let format = input.outputFormat(forBus: 0) + // 0 Hz / 0 ch (no input device) would make installTap raise an ObjC + // NSException that Swift can't catch — guard it out. + guard format.sampleRate > 0, format.channelCount > 0 else { + throw CaptureRestartError(message: "No audio input device available.") + } + guard let desiredFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: Double(WhisperKit.sampleRate), + channels: 1, + interleaved: false + ), let converter = AVAudioConverter(from: format, to: desiredFormat) else { + throw CaptureRestartError( + message: "mic format \(format) can't convert to WhisperKit's 16 kHz mono") + } + let bufferSize = AVAudioFrameCount(processor.minBufferLength) + input.installTap(onBus: 0, bufferSize: bufferSize, format: format) { [weak processor] buffer, _ in + guard let processor else { return } + var resampled = buffer + if !buffer.format.sampleRate.isEqual(to: Double(WhisperKit.sampleRate)) { + guard let converted = try? AudioProcessor.resampleBuffer(buffer, with: converter) + else { return } // drop an unconvertible buffer + resampled = converted + } + var samples = AudioProcessor.convertBufferToArray(buffer: resampled) + if processor.isInputSuppressed { + samples = [Float](repeating: 0, count: samples.count) + } + processor.processBuffer(samples) + } + engine.prepare() + try engine.start() + processor.audioEngine = engine + } + /// Ignore stop-time tails shorter than this (0.25 s @ 16 kHz): releasing the /// hotkey right at a decode boundary leaves a sliver of silence that isn't /// worth a decode pass. diff --git a/OpenWhisp/Services/WhisperKitStreamingEngine.swift b/OpenWhisp/Services/WhisperKitStreamingEngine.swift index 7c0f3e1..42e25bf 100644 --- a/OpenWhisp/Services/WhisperKitStreamingEngine.swift +++ b/OpenWhisp/Services/WhisperKitStreamingEngine.swift @@ -1,5 +1,6 @@ import Foundation -import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) +import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) +import AVFoundation // AVAudioEngine (config-change observer on WhisperKit's capture engine) /// Experimental real-time WhisperKit engine (streaming partials). /// @@ -13,7 +14,11 @@ import CoreAudio // AudioDeviceID (the input-device id threaded to WhisperKit) /// /// **Build:** real implementation only under `#if WHISPERKIT`; otherwise a stub /// that reports unavailability, so the default build is unaffected. -final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { +/// +/// NSObject so the AVAudioEngineConfigurationChange observer can be +/// selector-based (the block API takes a hard `@Sendable` closure whose +/// captures here are non-Sendable) — same shape as ParakeetStreamingEngine. +final class WhisperKitStreamingEngine: NSObject, StreamingTranscriptionEngine { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? var onError: ((String) -> Void)? @@ -36,6 +41,7 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { init(modelName: String = "openai_whisper-small") { self.modelName = modelName + super.init() } #if WHISPERKIT @@ -74,6 +80,19 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { /// re-press). See `SerialTaskChain`. @MainActor private let lifecycle = SerialTaskChain() + /// What the mid-session configuration-change restart needs (see + /// `armConfigChangeObserver`). Non-nil exactly while the observer is armed; + /// cleared on every teardown path. Unlike the engines that own their tap, + /// the capture rebuild itself lives on the stream handle + /// (`WhisperKitStreamHandle.restartCapture`) because the mic belongs to + /// WhisperKit's `AudioProcessor`. + @MainActor private var restartContext: RestartContext? + + private struct RestartContext { + let callbacks: SessionCallbacks + let myGeneration: Int + } + func start(language: String, prompt: String) throws { let task = WhisperKitTaskMapper.map(languageSetting: language) // Enqueue SYNCHRONOUSLY on the main actor so call order == enqueue order @@ -122,29 +141,13 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { // Resolve the selected input device. It's passed straight into // AudioStreamTranscriber(inputDeviceID:) (our WhisperKit fork backports // upstream #503), which forwards it to startRecordingLive — WhisperKit - // captures the chosen device directly, no system-default swap needed. A - // pinned but DISCONNECTED device falls back to the system default so the - // session can run (the AirPods-disconnect hang); a CONNECTED device that - // fails to resolve at application time stays a hard error. + // captures the chosen device directly, no system-default swap needed. let inputDeviceID: AudioDeviceID? - switch AudioInputRoutingPolicy.decide( - microphoneID: selectedDeviceID, - deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) - ) { - case .systemDefault: - inputDeviceID = nil - case .useDevice(let uid): - // Resolve to the concrete device id. A nil here (device vanished - // between canResolve and now) is treated as unresolved — a hard error, - // NOT a silent nil that would fall back to the system default. - guard let id = AudioInputRouter.resolve(uid: uid)?.deviceID else { - callbacks.error?(AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) - return - } - inputDeviceID = id - case .fallbackToDefault(let uid): - NSLog("[WhisperKitStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) - inputDeviceID = nil + do { + inputDeviceID = try resolveInputDeviceID() + } catch let setupError as CaptureSetupError { + callbacks.error?(setupError.message) + return } let kit = try await ensureLoaded() @@ -178,6 +181,9 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { } catch { NSLog("[WhisperKitStream] stream error: %@", error.localizedDescription) guard let self, self.transcriber === handle, !self.didFinish else { return } + // The stream is dead — a config-change restart would feed a + // torn-down decode loop, so disarm before erroring out. + self.removeConfigChangeObserver() callbacks.error?("WhisperKit streaming failed: \(error.localizedDescription)") } } @@ -211,6 +217,7 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { /// before any queued start proceeds. @MainActor private func runStop(cancel: Bool, final deliverFinal: ((String) -> Void)?) async { + removeConfigChangeObserver() let handle = transcriber transcriber = nil didFinish = true @@ -237,10 +244,16 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { private func handleState(_ state: WhisperKitStreamState, callbacks: SessionCallbacks) { // First state diff for this stream = capture is live (tap installed, // buffers flowing). Fires before any partial from the same diff so the - // session leaves arming before text starts arriving. + // session leaves arming before text starts arriving. This is also the + // earliest moment WhisperKit's capture AVAudioEngine exists, so the + // configuration-change observer arms here (the caller's generation + // guard makes self.generation this stream's own). if !startedNotified { startedNotified = true callbacks.started?() + if let engine = transcriber?.captureAudioEngine { + armConfigChangeObserver(for: engine, callbacks: callbacks, myGeneration: generation) + } } if let level = state.peakEnergy { // vadLevel is the absolute-curve reading; the display level's @@ -260,6 +273,108 @@ final class WhisperKitStreamingEngine: StreamingTranscriptionEngine { callbacks.partial?(text) } + /// Setup failure with a user-facing message (routing refused, device + /// vanished). `LocalizedError` so a generic `localizedDescription` still + /// reads as the message. + private struct CaptureSetupError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// Resolve `selectedDeviceID` to a concrete CoreAudio device id (nil = + /// system default). Shared by session start and the mid-session + /// configuration-change restart. A pinned but DISCONNECTED device falls + /// back to the system default so the session can run (the + /// AirPods-disconnect hang); a CONNECTED device that fails to resolve at + /// application time stays a hard error — NOT a silent nil that would fall + /// back to the system default. + @MainActor + private func resolveInputDeviceID() throws -> AudioDeviceID? { + switch AudioInputRoutingPolicy.decide( + microphoneID: selectedDeviceID, + deviceResolved: AudioInputRouter.canResolve(uid: selectedDeviceID) + ) { + case .systemDefault: + return nil + case .useDevice(let uid): + guard let id = AudioInputRouter.resolve(uid: uid)?.deviceID else { + throw CaptureSetupError(message: AudioInputRoutingPolicy.unresolvedMessage(uid: uid)) + } + return id + case .fallbackToDefault(let uid): + NSLog("[WhisperKitStreamingEngine] pinned mic '%@' disconnected — capturing system default", uid) + return nil + } + } + + /// Watch for the input device disconnecting / switching / renegotiating its + /// format mid-session. AVAudioEngine STOPS rendering when that happens, so + /// without this the session keeps showing "Listening…" while capturing + /// nothing — every word after the change is silently lost. Capture is + /// REBUILT into the same decode buffer (`WhisperKitStreamHandle. + /// restartCapture`): the realtime loop keeps everything already captured + /// and the dictation continues, losing only the glitch itself. Rebuild + /// failure (e.g. no input device left) surfaces on the session error path. + /// Same pattern as ParakeetStreamingEngine. + @MainActor + private func armConfigChangeObserver( + for engine: AVAudioEngine, + callbacks: SessionCallbacks, + myGeneration: Int + ) { + removeConfigChangeObserver() + restartContext = RestartContext(callbacks: callbacks, myGeneration: myGeneration) + // Selector-based on purpose: the block observer API takes a hard + // `@Sendable` closure, and every capture it needs (self, the session + // callbacks) is non-Sendable — the selector target keeps the compile + // warning-free and the context lives in `restartContext` instead. + NotificationCenter.default.addObserver( + self, + selector: #selector(inputConfigurationDidChange(_:)), + name: .AVAudioEngineConfigurationChange, + object: engine) + } + + @MainActor + private func removeConfigChangeObserver() { + NotificationCenter.default.removeObserver( + self, name: .AVAudioEngineConfigurationChange, object: nil) + restartContext = nil + } + + /// Fires on the notification-posting thread; hop to the main actor where + /// the session state lives. Same hop shape as the state callback. + @objc private func inputConfigurationDidChange(_ note: Notification) { + DispatchQueue.main.async { + MainActor.assumeIsolated { self.handleInputConfigurationChange() } + } + } + + @MainActor + private func handleInputConfigurationChange() { + // Same session fence as the state callback: a stop or a newer session + // owns the mic now — touch nothing. (Teardown also clears the context.) + guard let context = restartContext, CaptureConfigChangePolicy.action( + generationMatches: generation == context.myGeneration, + didStop: didFinish + ) == .restartCapture, let handle = transcriber else { return } + NSLog("[WhisperKitStream] input device configuration changed — restarting capture") + removeConfigChangeObserver() + do { + let deviceID = try resolveInputDeviceID() + try handle.restartCapture(inputDeviceID: deviceID) + // Re-arm on the REPLACEMENT engine — the old observer died with the + // stale engine object. + if let engine = handle.captureAudioEngine { + armConfigChangeObserver( + for: engine, callbacks: context.callbacks, myGeneration: context.myGeneration) + } + } catch { + context.callbacks.error?(CaptureConfigChangePolicy.restartFailedMessage( + underlying: error.localizedDescription)) + } + } + @discardableResult private func ensureLoaded() async throws -> WhisperKitHandle { if let kit = loadedKit { return kit } diff --git a/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift b/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift index 0bd5c21..0a77a00 100644 --- a/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift +++ b/Tests/OpenWhispCoreTests/CaptureConfigChangePolicyTests.swift @@ -2,7 +2,8 @@ import XCTest @testable import OpenWhispCore /// Pins the gates for the mid-session `AVAudioEngineConfigurationChange` -/// recovery (Parakeet streaming): a live session restarts capture; a stale or +/// recovery shared by ALL streaming capture engines (Parakeet, WhisperKit, +/// Apple Speech, SpeechAnalyzer): a live session restarts capture; a stale or /// stopped session ignores the notification. Without the restart, a device /// disconnect/format renegotiation leaves the session showing "Listening…" /// while capturing NOTHING — words are silently lost for the session's