diff --git a/OpenWhisp/Services/WhisperKitBridge.swift b/OpenWhisp/Services/WhisperKitBridge.swift index 9623ed5..c758525 100644 --- a/OpenWhisp/Services/WhisperKitBridge.swift +++ b/OpenWhisp/Services/WhisperKitBridge.swift @@ -53,10 +53,33 @@ public enum WhisperKitStreamingDecodePolicy { public static let windowSamples = 480_000 public static let sampleRate = 16_000 - /// `AudioStreamTranscriber`'s default `requiredSegmentsForConfirmation`. We - /// don't override it at the call site, so this is the live value. + /// `AudioStreamTranscriber`'s default `requiredSegmentsForConfirmation` — + /// the live value for plain TRANSCRIBE sessions (we don't override it). public static let requiredSegmentsForConfirmation = 2 + /// The confirmation threshold `makeStreamHandle` passes for a session, + /// by task. + /// + /// This is the streaming loop's dominant latency knob: every decode pass + /// re-decodes from `lastConfirmedSegmentEndSeconds`, and confirmation + /// always holds back the newest `requiredSegmentsForConfirmation` + /// segments. Holding back 2 means the re-decoded tail settles at ~2–3 + /// segments of audio; on the TRANSLATE task each of those passes also has + /// to re-GENERATE English for that whole tail (more tokens, lower + /// confidence, more fallbacks than same-language transcribe), so after + /// ~30–40 s of continuous speech the pass cost visibly outgrows the ≥1 s + /// new-audio cadence and partials start lagging. + /// + /// Translate therefore holds back only 1 segment: the clip point advances + /// a full segment sooner, roughly halving the steady-state re-decode + /// window. The cost is that a segment freezes (is confirmed) one window + /// earlier — an acceptable trade on the translate path, where the live + /// preview is already a moving translation rather than verbatim text. + /// Transcribe keeps the upstream default of 2. + public static func requiredSegmentsForConfirmation(translate: Bool) -> Int { + translate ? 1 : requiredSegmentsForConfirmation + } + /// Segments a window yields, given whether timestamp tokens were emitted. /// Without timestamps the seeker can't split, so the window is one segment. public static func segmentsPerWindow(timestampsEmitted: Bool) -> Int { @@ -352,6 +375,12 @@ enum WhisperKitBridge { tokenizer: tokenizer, audioProcessor: kit.audioProcessor, decodingOptions: options, + // Task-aware confirmation lag — the streaming loop's dominant + // latency knob (see WhisperKitStreamingDecodePolicy): translate + // holds back 1 segment instead of 2 so the re-decoded tail stays + // short and partials keep pace past the ~30–40 s mark. + requiredSegmentsForConfirmation: + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: task.translate), useVAD: true, // skip silence — don't transcribe dead air inputDeviceID: inputDeviceID, // nil = system default (fork backport of #503) stateChangeCallback: { _, new in diff --git a/Tests/OpenWhispCoreTests/WhisperKitStreamingDecodePolicyTests.swift b/Tests/OpenWhispCoreTests/WhisperKitStreamingDecodePolicyTests.swift index 06197f8..f247694 100644 --- a/Tests/OpenWhispCoreTests/WhisperKitStreamingDecodePolicyTests.swift +++ b/Tests/OpenWhispCoreTests/WhisperKitStreamingDecodePolicyTests.swift @@ -68,4 +68,46 @@ final class WhisperKitStreamingDecodePolicyTests: XCTestCase { XCTAssertEqual(WhisperKitStreamingDecodePolicy.sampleRate, 16_000) XCTAssertEqual(WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation, 2) } + + // MARK: - Task-aware confirmation lag (translate latency) + + /// The translate-latency fix: translate holds back only ONE segment, so the + /// clip point trails the speech by one segment instead of two and every + /// pass re-decodes (re-translates) a much shorter tail. Transcribe keeps + /// the upstream default — its latency was never the complaint, and the + /// extra held-back segment buys verbatim text more revision room. + func testTranslateHoldsBackOneSegmentTranscribeTwo() { + XCTAssertEqual( + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: true), 1, + "Translate re-GENERATES English for the whole unconfirmed tail every pass — " + + "holding back 2 segments makes passes outgrow the 1 s cadence after " + + "~30-40 s of continuous speech.") + XCTAssertEqual( + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: false), + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation, + "Transcribe stays on AudioStreamTranscriber's upstream default.") + } + + /// The #222 invariant must survive the translate threshold: a timestamped + /// window still yields MORE segments than translate holds back, so the + /// confirmed end keeps advancing and dictation length stays unbounded. + func testTranslateThresholdStillAdvancesTheConfirmedEnd() { + XCTAssertGreaterThan( + WhisperKitStreamingDecodePolicy.segmentsPerWindow(timestampsEmitted: true), + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: true), + "If translate's threshold ever reaches the per-window segment count, " + + "confirmation stalls and translate sessions truncate at 30 s — " + + "the exact bug #222 fixed for transcribe.") + } + + /// A threshold below 1 would confirm EVERY segment immediately, leaving no + /// unconfirmed tail at all — the decoder could never revise anything and + /// half-heard words would freeze into the transcript. Both tasks must hold + /// back at least one segment. + func testBothTasksHoldBackAtLeastOneSegment() { + XCTAssertGreaterThanOrEqual( + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: true), 1) + XCTAssertGreaterThanOrEqual( + WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate: false), 1) + } } diff --git a/docs/WHISPERKIT_PILOT.md b/docs/WHISPERKIT_PILOT.md index 37768d9..beace36 100644 --- a/docs/WHISPERKIT_PILOT.md +++ b/docs/WHISPERKIT_PILOT.md @@ -16,8 +16,11 @@ into the same session machinery as Apple Speech: live partials drive the overlay preview, and the full transcript is pasted on release. Key decode settings (`WhisperKitBridge.makeStreamHandle`): -- `skipSpecialTokens: true` + `withoutTimestamps: true` — streaming segment text is - the raw token stream otherwise (`<|startoftranscript|>…`). +- `skipSpecialTokens: true` — streaming segment text is the raw token stream + otherwise (`<|startoftranscript|>…`). `withoutTimestamps` MUST stay `false` + (`WhisperKitStreamingDecodePolicy.withoutTimestamps`): timestamp tokens are what + let segments split and confirm, advancing the decode window — suppressing them + truncated every dictation at 30 s (fixed in #222). - `detectLanguage: true` for the "auto" case — WhisperKit's prefill otherwise forces English, so Russian came out translated. Explicit-language / translate skip this. @@ -146,8 +149,10 @@ a 16 GB Mac — `small` is the default for that reason. ## Known limitations of this pilot -- **File transcription only** — no true streaming partials yet (that's the next - step; WhisperKit supports it via LocalAgreement). +- ~~**File transcription only**~~ — streaming partials shipped since + (`WhisperKitStreamingEngine` / `AudioStreamTranscriber`); this pilot doc predates + them. Streaming latency is governed by the confirmation lag — see + `WhisperKitStreamingDecodePolicy.requiredSegmentsForConfirmation(translate:)`. - **Custom vocabulary prompt is not wired** for WhisperKit. WhisperKit biases via `promptTokens: [Int]?` (token IDs), not a plain string, so the vocabulary feature only affects the whisper.cpp backend in this pilot.