Skip to content

degraded listener POC #3272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
yujonglee wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

degraded listener POC #3272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

103 changes: 88 additions & 15 deletions apps/desktop/src/components/main/body/sessions/outer-header/listen.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { useHover } from "@uidotdev/usehooks";
import { MicOff } from "lucide-react";
import { useCallback, useEffect, useRef } from "react";
import { AlertTriangle, MicOff } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef } from "react";

import type { DegradedError } from "@hypr/plugin-listener";
import {
Tooltip,
TooltipContent,
Expand Down Expand Up @@ -188,24 +189,58 @@ function StartButton({ sessionId }: { sessionId: string }) {
);
}

function formatDegradedError(error: DegradedError): string {
switch (error.type) {
case "authentication_failed":
return `Authentication failed for ${error.provider}`;
case "upstream_unavailable":
return error.message;
case "connection_timeout":
return "Connection timed out";
case "stream_error":
return error.message;
case "channel_overflow":
return "Audio channel overflow";
default:
return "Transcription unavailable";
}
}

function InMeetingIndicator({ sessionId }: { sessionId: string }) {
const [ref, hovered] = useHover();
const openNew = useTabs((state) => state.openNew);

const { mode, stop, amplitude, muted } = useListener((state) => ({
mode: state.getSessionMode(sessionId),
stop: state.stop,
amplitude: state.live.amplitude,
muted: state.live.muted,
}));
const { mode, stop, amplitude, muted, degradedError } = useListener(
(state) => ({
mode: state.getSessionMode(sessionId),
stop: state.stop,
amplitude: state.live.amplitude,
muted: state.live.muted,
degradedError: state.live.degradedError,
}),
);

const active = mode === "active" || mode === "finalizing";
const finalizing = mode === "finalizing";
const isDegraded = !!degradedError;

const degradedMessage = useMemo(
() =>
degradedError
? `Transcription degraded: ${formatDegradedError(degradedError)}`
: null,
[degradedError],
);

const handleConfigureAction = useCallback(() => {
openNew({ type: "ai", state: { tab: "transcription" } });
}, [openNew]);

if (!active) {
return null;
}

return (
const button = (
<button
ref={ref}
type="button"
Expand All @@ -215,11 +250,22 @@ function InMeetingIndicator({ sessionId }: { sessionId: string }) {
"inline-flex items-center justify-center rounded-md text-sm font-medium",
finalizing
? ["text-neutral-500", "bg-neutral-100", "cursor-wait"]
: ["text-red-500 hover:text-red-600", "bg-red-50 hover:bg-red-100"],
: isDegraded
? [
"text-amber-600 hover:text-amber-700",
"bg-amber-50 hover:bg-amber-100",
]
: ["text-red-500 hover:text-red-600", "bg-red-50 hover:bg-red-100"],
"w-28.5 h-7",
"disabled:pointer-events-none disabled:opacity-50",
])}
title={finalizing ? "Finalizing" : "Stop listening"}
title={
finalizing
? "Finalizing"
: isDegraded
? (degradedMessage ?? "Transcription degraded")
: "Stop listening"
}
aria-label={finalizing ? "Finalizing" : "Stop listening"}
>
{finalizing ? (
Expand All @@ -234,12 +280,13 @@ function InMeetingIndicator({ sessionId }: { sessionId: string }) {
hovered ? "hidden" : "flex",
])}
>
{muted && <MicOff size={14} />}
{isDegraded && <AlertTriangle size={14} />}
{muted && !isDegraded && <MicOff size={14} />}
<ScrollingWaveform
amplitude={(amplitude.mic + amplitude.speaker) / 2}
color="#ef4444"
color={isDegraded ? "#d97706" : "#ef4444"}
height={26}
width={muted ? 68 : 88}
width={muted || isDegraded ? 68 : 88}
barWidth={2}
gap={1}
minBarHeight={2}
Expand All @@ -252,11 +299,37 @@ function InMeetingIndicator({ sessionId }: { sessionId: string }) {
hovered ? "flex" : "hidden",
])}
>
<span className="w-3 h-3 bg-red-500 rounded-none" />
<span
className={cn([
"w-3 h-3 rounded-none",
isDegraded ? "bg-amber-500" : "bg-red-500",
])}
/>
<span>Stop</span>
</div>
</>
)}
</button>
);

if (!isDegraded) {
return button;
}

return (
<Tooltip delayDuration={0}>
<TooltipTrigger asChild>
<span className="inline-block">{button}</span>
</TooltipTrigger>
<TooltipContent side="bottom">
<ActionableTooltipContent
message={degradedMessage ?? "Transcription degraded"}
action={{
label: "Configure",
handleClick: handleConfigureAction,
}}
/>
</TooltipContent>
</Tooltip>
);
}
16 changes: 13 additions & 3 deletions apps/desktop/src/components/main/body/sessions/shared.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export function RecordingIcon() {

export function useListenButtonState(sessionId: string) {
const sessionMode = useListener((state) => state.getSessionMode(sessionId));
const lastError = useListener((state) => state.live.lastError);
const degradedError = useListener((state) => state.live.degradedError);
const active = sessionMode === "active" || sessionMode === "finalizing";
const batching = sessionMode === "running_batch";

Expand All @@ -81,8 +81,18 @@ export function useListenButtonState(sessionId: string) {
isOfflineWithCloudModel;

let warningMessage = "";
if (lastError) {
warningMessage = `Session failed: ${lastError}`;
if (degradedError) {
const errorMessage =
degradedError.type === "authentication_failed"
? `Authentication failed for ${degradedError.provider}`
: degradedError.type === "upstream_unavailable"
? degradedError.message
: degradedError.type === "connection_timeout"
? "Connection timed out"
: degradedError.type === "stream_error"
? degradedError.message
: "Transcription unavailable";
warningMessage = `Transcription degraded: ${errorMessage}`;
} else if (isLocalServerLoading) {
warningMessage = "Local STT server is starting up...";
} else if (isOfflineWithCloudModel) {
Expand Down
41 changes: 7 additions & 34 deletions apps/desktop/src/store/zustand/listener/general.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import { commands as detectCommands } from "@hypr/plugin-detect";
import { commands as hooksCommands } from "@hypr/plugin-hooks";
import { commands as iconCommands } from "@hypr/plugin-icon";
import {
type DegradedError,
commands as listenerCommands,
events as listenerEvents,
type SessionDataEvent,
type SessionErrorEvent,
type SessionLifecycleEvent,
type SessionParams,
type SessionProgressEvent,
Expand Down Expand Up @@ -49,7 +49,7 @@ export type GeneralState = {
intervalId?: NodeJS.Timeout;
sessionId: string | null;
muted: boolean;
lastError: string | null;
degradedError: DegradedError | null;
device: string | null;
};
};
Expand Down Expand Up @@ -77,15 +77,14 @@ const initialState: GeneralState = {
seconds: 0,
sessionId: null,
muted: false,
lastError: null,
degradedError: null,
device: null,
},
};

type EventListeners = {
lifecycle: (payload: SessionLifecycleEvent) => void;
progress: (payload: SessionProgressEvent) => void;
error: (payload: SessionErrorEvent) => void;
data: (payload: SessionDataEvent) => void;
};

Expand All @@ -101,9 +100,6 @@ const listenToAllSessionEvents = (
listenerEvents.sessionProgressEvent.listen(({ payload }) =>
handlers.progress(payload),
),
listenerEvents.sessionErrorEvent.listen(({ payload }) =>
handlers.error(payload),
),
listenerEvents.sessionDataEvent.listen(({ payload }) =>
handlers.data(payload),
),
Expand Down Expand Up @@ -184,6 +180,7 @@ export const createGeneralSlice = <
draft.live.seconds = 0;
draft.live.intervalId = intervalId;
draft.live.sessionId = targetSessionId;
draft.live.degradedError = payload.error ?? null;
}),
);
} else if (payload.type === "finalizing") {
Expand Down Expand Up @@ -212,7 +209,7 @@ export const createGeneralSlice = <
draft.live.loadingPhase = "idle";
draft.live.sessionId = null;
draft.live.eventUnlisteners = undefined;
draft.live.lastError = payload.error ?? null;
draft.live.degradedError = null;
draft.live.device = null;
}),
);
Expand All @@ -230,7 +227,7 @@ export const createGeneralSlice = <
set((state) =>
mutate(state, (draft) => {
draft.live.loadingPhase = "audio_initializing";
draft.live.lastError = null;
draft.live.degradedError = null;
}),
);
} else if (payload.type === "audio_ready") {
Expand All @@ -255,29 +252,6 @@ export const createGeneralSlice = <
}
};

const handleErrorEvent = (payload: SessionErrorEvent) => {
if (payload.session_id !== targetSessionId) {
return;
}

if (payload.type === "audio_error") {
set((state) =>
mutate(state, (draft) => {
draft.live.lastError = payload.error;
if (payload.is_fatal) {
draft.live.loading = false;
}
}),
);
} else if (payload.type === "connection_error") {
set((state) =>
mutate(state, (draft) => {
draft.live.lastError = payload.error;
}),
);
}
};

const handleDataEvent = (payload: SessionDataEvent) => {
if (payload.session_id !== targetSessionId) {
return;
Expand Down Expand Up @@ -308,7 +282,6 @@ export const createGeneralSlice = <
const unlisteners = yield* listenToAllSessionEvents({
lifecycle: handleLifecycleEvent,
progress: handleProgressEvent,
error: handleErrorEvent,
data: handleDataEvent,
});

Expand Down Expand Up @@ -384,7 +357,7 @@ export const createGeneralSlice = <
draft.live.seconds = 0;
draft.live.sessionId = null;
draft.live.muted = initialState.live.muted;
draft.live.lastError = null;
draft.live.degradedError = null;
draft.live.device = null;
}),
);
Expand Down
1 change: 0 additions & 1 deletion plugins/listener/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ hound = { workspace = true }
vorbis_rs = { workspace = true }

ractor = { workspace = true }
ractor-supervisor = { workspace = true }

futures-util = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
Expand Down
7 changes: 3 additions & 4 deletions plugins/listener/js/bindings.gen.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,10 @@ async listDocumentedLanguageCodesLive() : Promise<Result<string[], string>> {

export const events = __makeEvents__<{
sessionDataEvent: SessionDataEvent,
sessionErrorEvent: SessionErrorEvent,
sessionLifecycleEvent: SessionLifecycleEvent,
sessionProgressEvent: SessionProgressEvent
}>({
sessionDataEvent: "plugin:listener:session-data-event",
sessionErrorEvent: "plugin:listener:session-error-event",
sessionLifecycleEvent: "plugin:listener:session-lifecycle-event",
sessionProgressEvent: "plugin:listener:session-progress-event"
})
Expand All @@ -109,9 +107,10 @@ sessionProgressEvent: "plugin:listener:session-progress-event"

/** user-defined types **/

export type CriticalError = { message: string }
export type DegradedError = { type: "authentication_failed"; provider: string } | { type: "upstream_unavailable"; message: string } | { type: "connection_timeout" } | { type: "stream_error"; message: string } | { type: "channel_overflow" }
export type SessionDataEvent = { type: "audio_amplitude"; session_id: string; mic: number; speaker: number } | { type: "mic_muted"; session_id: string; value: boolean } | { type: "stream_response"; session_id: string; response: StreamResponse }
export type SessionErrorEvent = { type: "audio_error"; session_id: string; error: string; device: string | null; is_fatal: boolean } | { type: "connection_error"; session_id: string; error: string }
export type SessionLifecycleEvent = { type: "inactive"; session_id: string; error: string | null } | { type: "active"; session_id: string } | { type: "finalizing"; session_id: string }
export type SessionLifecycleEvent = { type: "inactive"; session_id: string; error?: CriticalError | null } | { type: "active"; session_id: string; error?: DegradedError | null } | { type: "finalizing"; session_id: string }
export type SessionParams = { session_id: string; languages: string[]; onboarding: boolean; record_enabled: boolean; model: string; base_url: string; api_key: string; keywords: string[] }
export type SessionProgressEvent = { type: "audio_initializing"; session_id: string } | { type: "audio_ready"; session_id: string; device: string | null } | { type: "connecting"; session_id: string } | { type: "connected"; session_id: string; adapter: string }
export type StreamAlternatives = { transcript: string; words: StreamWord[]; confidence: number; languages?: string[] }
Expand Down
Loading
Loading