diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 7703f6c09f..18e2eb373f 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -258,6 +258,7 @@ export const de: Record = { "dash.backendAuto": "Automatisch", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "Sidecar-Einstellungen gespeichert. Angewendet bei der nächsten Anfrage.", "dash.sidecarSaveFailed": "Sidecar-Einstellungen konnten nicht gespeichert werden.", "dash.injectionLabel": "Sub-Agent-Delegation", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 9ae8861249..6269347daa 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -270,6 +270,7 @@ export const en = { "dash.backendAuto": "Auto", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "Sidecar settings saved. Applied on the next request.", "dash.sidecarSaveFailed": "Failed to save sidecar settings.", "dash.injectionLabel": "Sub-agent delegation", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index d5f7a915b4..3a03292128 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -267,6 +267,7 @@ export const ja: Record = { "dash.backendAuto": "自動", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "サイドカー設定を保存しました。次回リクエスト時に適用されます。", "dash.sidecarSaveFailed": "サイドカー設定の保存に失敗しました。", "dash.injectionLabel": "サブエージェント委任", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index e0ee1af6a4..0e62048f7a 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -262,6 +262,7 @@ export const ko: Record = { "dash.backendAuto": "자동", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "사이드카 설정이 저장됐습니다. 다음 요청부터 적용됩니다.", "dash.sidecarSaveFailed": "사이드카 설정 저장에 실패했습니다.", "dash.injectionLabel": "서브에이전트 위임", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index f707bb5f5e..c0451b5877 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -267,6 +267,7 @@ export const ru: Record = { "dash.backendAuto": "Авто", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "Настройки сайдкара сохранены. Вступят в силу со следующего запроса.", "dash.sidecarSaveFailed": "Не удалось сохранить настройки сайдкара.", "dash.injectionLabel": "Делегирование подагентам", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 14113c6110..4f8dec6f82 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -262,6 +262,7 @@ export const zh: Record = { "dash.backendAuto": "自动", "dash.backendOpenAI": "OpenAI", "dash.backendAnthropic": "Anthropic", + "dash.backendChat": "Chat API", "dash.sidecarSaved": "附属设置已保存。将在下一个请求时生效。", "dash.sidecarSaveFailed": "保存附属设置失败。", "dash.injectionLabel": "子代理委托", diff --git a/gui/src/pages/claude-code-sections.tsx b/gui/src/pages/claude-code-sections.tsx index 70fa292fa4..183d23b3f5 100644 --- a/gui/src/pages/claude-code-sections.tsx +++ b/gui/src/pages/claude-code-sections.tsx @@ -170,6 +170,7 @@ export function ClaudeCodeSettingsCard({ { value: "auto", label: t("dash.backendAuto") }, { value: "openai", label: t("dash.backendOpenAI") }, { value: "anthropic", label: t("dash.backendAnthropic") }, + ...(key === "visionSidecar" ? [{ value: "chat", label: t("dash.backendChat") }] : []), ]} onChange={value => { // Auto may exist as an empty in-memory draft so the model input diff --git a/gui/src/pages/claude-manual-env.ts b/gui/src/pages/claude-manual-env.ts index bb0c2970dc..80373b26e1 100644 --- a/gui/src/pages/claude-manual-env.ts +++ b/gui/src/pages/claude-manual-env.ts @@ -4,7 +4,7 @@ * copy-paste shell block is directly unit-testable (tests/claude-manual-env.test.ts). */ -export type SidecarBackend = "openai" | "anthropic"; +export type SidecarBackend = "openai" | "anthropic" | "chat"; export interface SidecarOverride { backend?: SidecarBackend; model?: string } export interface ClaudeManualEnvState { diff --git a/gui/src/pages/dashboard-overview-sections.tsx b/gui/src/pages/dashboard-overview-sections.tsx index 1b74a7021f..bc3dd1aa44 100644 --- a/gui/src/pages/dashboard-overview-sections.tsx +++ b/gui/src/pages/dashboard-overview-sections.tsx @@ -274,7 +274,7 @@ export function DashboardMaintenancePanel({ d }: { d: Dash }) { export function DashboardSidecarPanels({ d }: { d: Dash }) { const { t, settings, settingsSaving, toggleCodexAutoStart, - sidecar, sidecarSaving, sidecarModels, models, saveSidecar, + sidecar, sidecarSaving, sidecarModels, visionSidecarModels, models, saveSidecar, shadowCall, shadowCallSaving, shadowCallHelpTriggerRef, shadowCallHelpOpen, setShadowCallHelpOpen, saveShadowCall, } = d; @@ -317,9 +317,35 @@ export function DashboardSidecarPanels({ d }: { d: Dash }) {
{t("dash.visionSidecar")}
+ { void saveSidecar({ vision: { model, backend: sidecarBackendForModel(models, model) } }); }} disabled={!sidecar || sidecarSaving} label={t("dash.sidecarModel")} diff --git a/gui/src/pages/dashboard-shared.ts b/gui/src/pages/dashboard-shared.ts index 6248e19b55..6431993392 100644 --- a/gui/src/pages/dashboard-shared.ts +++ b/gui/src/pages/dashboard-shared.ts @@ -39,7 +39,7 @@ export async function requireJson(res: Response, fallbackMessage?: string): P export interface HealthData { status: string; version: string; uptime: number } export interface ProviderInfo { name: string; adapter: string; baseUrl: string; defaultModel?: string; hasApiKey: boolean } -export interface ModelInfo { id: string; provider: string; namespaced: string; owned_by?: string } +export interface ModelInfo { id: string; provider: string; namespaced: string; owned_by?: string; disabled?: boolean } export interface SettingsData { codexAutoStart: boolean; port: number; @@ -54,7 +54,7 @@ export interface SettingsData { diagnosticStale: boolean; }; } -export type SidecarBackend = "openai" | "anthropic"; +export type SidecarBackend = "openai" | "anthropic" | "chat"; export interface SidecarSetting { backend?: SidecarBackend; model: string } export interface SidecarData { webSearch: SidecarSetting; vision: SidecarSetting } export interface SidecarPatch { @@ -161,6 +161,20 @@ export function sidecarModelOptions(models: ModelInfo[]) { return out; } +/** Vision can use enabled routed providers; namespaced values preserve provider selection. */ +export function visionSidecarModelOptions(models: ModelInfo[]) { + const out: Array<{ value: string; label: string }> = []; + for (const model of models) { + if (model.disabled === true) continue; + if (model.provider === "openai" || model.provider === "anthropic") { + out.push({ value: model.id, label: `${model.provider}/${model.id}` }); + } else { + out.push({ value: model.namespaced, label: model.namespaced }); + } + } + return out; +} + /** Options for shadow-call replacement models use the proxy's canonical routing id. */ export function shadowCallModelOptions(models: ModelInfo[], current: string | undefined) { const out = [{ value: "", label: "—" }, ...models.map(model => ({ value: model.namespaced, label: model.namespaced }))]; @@ -169,7 +183,10 @@ export function shadowCallModelOptions(models: ModelInfo[], current: string | un } export function sidecarBackendForModel(models: ModelInfo[], modelId: string): SidecarBackend { - return models.find(model => model.id === modelId)?.provider === "anthropic" ? "anthropic" : "openai"; + const model = models.find(item => item.id === modelId || item.namespaced === modelId); + if (model?.provider === "anthropic") return "anthropic"; + if (model?.provider === "openai") return "openai"; + return "chat"; } let lastInputWasKeyboard = false; diff --git a/gui/src/pages/use-dashboard-data.ts b/gui/src/pages/use-dashboard-data.ts index 0264781fa6..50f2e15ef7 100644 --- a/gui/src/pages/use-dashboard-data.ts +++ b/gui/src/pages/use-dashboard-data.ts @@ -46,6 +46,7 @@ import { readDashboardSectionFromHash, requireJson, sidecarModelOptions, + visionSidecarModelOptions, useModalDialog, } from "./dashboard-shared"; @@ -453,13 +454,20 @@ export function useDashboardData(apiBase: string) { }, [grouped, modelQuery]); const sidecarModels = useMemo(() => { const opts = sidecarModelOptions(models); - for (const id of [sidecar?.webSearch.model, sidecar?.vision.model]) { - if (id && !opts.some(option => option.value === id)) { - opts.unshift({ value: id, label: id }); - } + const current = sidecar?.webSearch.model; + const currentModel = models.find(model => model.id === current || model.namespaced === current); + if (current && (currentModel?.provider === "openai" || currentModel?.provider === "anthropic") + && !opts.some(option => option.value === current)) { + opts.unshift({ value: current, label: current }); } return opts; }, [models, sidecar]); + const visionSidecarModels = useMemo(() => { + const opts = visionSidecarModelOptions(models); + const current = sidecar?.vision.model; + if (current && !opts.some(option => option.value === current)) opts.unshift({ value: current, label: current }); + return opts; + }, [models, sidecar]); const saveSidecar = async (patch: SidecarPatch) => { if (!sidecar || sidecarSaving) return; @@ -745,7 +753,7 @@ export function useDashboardData(apiBase: string) { updateCheck, updateError, updateJob, reconnecting, error, effortCapHelpTriggerRef, updateTriggerRef, maHelpTriggerRef, shadowCallHelpTriggerRef, effortCapHelpDialogRef, updateDialogRef, maHelpDialogRef, shadowCallHelpDialogRef, - filteredGroups, sidecarModels, + filteredGroups, sidecarModels, visionSidecarModels, saveSidecar, saveShadowCall, switchMaMode, toggleCodexAutoStart, runSync, clearSyncFeedback, fetchUpdateCheck, closeUpdateDialog, openUpdateDialog, changeUpdateChannel, runUpdate, }; diff --git a/src/cli/agent.ts b/src/cli/agent.ts index 62e4f6c929..1c897baaab 100644 --- a/src/cli/agent.ts +++ b/src/cli/agent.ts @@ -20,7 +20,7 @@ const USAGE = `Usage: ocx agent effort [--main ] [--subagent ] [--json] ocx agent subagents [model,model...] [--json] ocx agent fallback [model,model...] [--poll-ms <5000-600000>] [--json] - ocx agent sidecar [--model ] [--backend ] + ocx agent sidecar [--model ] [--backend ] [--reasoning ] [--max-descriptions ] [--json]`; function clearable(value: string | undefined): string | null | undefined { diff --git a/src/cli/integrations.ts b/src/cli/integrations.ts index a3f7b69503..9e82a8e09c 100644 --- a/src/cli/integrations.ts +++ b/src/cli/integrations.ts @@ -19,7 +19,7 @@ const CLAUDE_USAGE = `Usage: [--compact-window ] [--inject-agents ] [--small-fast-model ] [--model-map ] [--blocked-skills ] [--web-model ] [--web-backend ] - [--vision-model ] [--vision-backend ] [--json]`; + [--vision-model ] [--vision-backend ] [--json]`; const GROK_USAGE = `Usage: ocx grok [status] [--json] diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index 17f7c00b32..0b1bc4e458 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -1004,8 +1004,9 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise if (section === undefined || section === null) continue; if (!isPlainObject(section)) return jsonResponse({ error: `${field} must be an object or null` }, 400); if (section.backend !== undefined && section.backend !== null - && section.backend !== "openai" && section.backend !== "anthropic") { - return jsonResponse({ error: `${field}.backend must be openai, anthropic, or null` }, 400); + && section.backend !== "openai" && section.backend !== "anthropic" && (field !== "visionSidecar" || section.backend !== "chat")) { + const accepted = field === "visionSidecar" ? "openai, anthropic, chat, or null" : "openai, anthropic, or null"; + return jsonResponse({ error: `${field}.backend must be ${accepted}` }, 400); } if (section.model !== undefined && typeof section.model !== "string") { return jsonResponse({ error: `${field}.model must be a string` }, 400); @@ -1019,8 +1020,8 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise delete next[field]; continue; } - const requested = section as { backend?: "openai" | "anthropic" | null; model?: string }; - const override: NonNullable = { ...next[field] }; + const requested = section as { backend?: "openai" | "anthropic" | "chat" | null; model?: string }; + const override = { ...(next[field] as Record | undefined) } as Record; if (requested.backend === null) delete override.backend; else if (requested.backend !== undefined) override.backend = requested.backend; if (requested.model === "") delete override.model; diff --git a/src/server/management/config-routes.ts b/src/server/management/config-routes.ts index 624836336b..0fea22bb0f 100644 --- a/src/server/management/config-routes.ts +++ b/src/server/management/config-routes.ts @@ -339,8 +339,8 @@ export async function handleConfigRoutes(ctx: ManagementContext): Promise { + return { type: "image_url", image_url: { url: imageUrl, detail: detail ?? "high" } }; +} + +async function describeImageGoogle( + imageUrl: string, + detail: string | undefined, + contextText: string, + provider: OcxProviderConfig, + providerName: string, + settings: ChatVisionSettings, + abortSignal?: AbortSignal, + recordOutcome?: SidecarOutcomeRecorder, +): Promise { + let requestProvider = provider; + if (provider.authMode === "oauth") { + try { + const token = await getValidAccessToken(providerName); + const project = provider.googleMode === "cloud-code-assist" + ? getOAuthCredentialProjectId(providerName) + : provider.project; + requestProvider = { ...provider, apiKey: token, ...(project ? { project } : {}) }; + } catch (e) { + return { text: "", error: `google oauth token failed: ${e instanceof Error ? e.message : String(e)}` }; + } + } + + const parsed = { + modelId: settings.model, + context: { + messages: [{ + role: "user", + content: [ + { type: "text", text: contextText || "Describe this image." }, + { type: "image", imageUrl, detail }, + ], + timestamp: Date.now(), + }], + }, + stream: true, + options: { maxOutputTokens: 1024, reasoning: "low" }, + } as OcxParsedRequest; + const budget = createTranslatorBudget(); + const adapter = createGoogleAdapter(requestProvider); + const sidecarExit = sidecarEnter("vision"); + try { + const request = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: budget, + abortSignal, + }); + const response = adapter.fetchResponse + ? await adapter.fetchResponse(request, { + abortSignal, + timeoutMs: settings.timeoutMs, + stream: true, + returnRawErrors: true, + }) + : await fetch(request.url, { + method: request.method, + headers: request.headers, + body: request.body, + signal: abortSignal, + }); + recordOutcome?.(response.status); + if (!response.ok) { + const text = await response.text().catch(() => ""); + return { text: "", error: `google sidecar HTTP ${response.status}: ${redactSecretString(text.slice(0, 200))}` }; + } + let text = ""; + for await (const event of adapter.parseStream(response, budget)) { + if (event.type === "text_delta") text += event.text; + if (event.type === "error") return { text: "", error: event.message }; + } + const trimmed = text.trim(); + if (!trimmed) return { text: "", error: "google sidecar produced no description" }; + return { text: trimmed }; + } catch (e) { + recordOutcome?.(e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error"); + return { text: "", error: e instanceof Error ? e.message : String(e) }; + } finally { + sidecarExit(); + } +} + +export async function describeImageChat( + imageUrl: string, + detail: string | undefined, + contextText: string, + provider: OcxProviderConfig, + providerName: string, + settings: ChatVisionSettings, + abortSignal?: AbortSignal, + recordOutcome?: SidecarOutcomeRecorder, +): Promise { + if (provider.adapter === "google") { + return describeImageGoogle( + imageUrl, + detail, + contextText, + provider, + providerName, + settings, + abortSignal, + recordOutcome, + ); + } + let authHeader = ""; + if (provider.authMode === "oauth") { + try { + const token = await getValidAccessToken(providerName); + authHeader = `Bearer ${token}`; + } catch (e) { + return { text: "", error: `oauth token failed: ${e instanceof Error ? e.message : String(e)}` }; + } + } else { + const apiKey = provider.apiKey ?? provider.apiKeyPool?.[0]?.key ?? provider.headers?.Authorization?.replace(/^Bearer\s+/i, ""); + if (!apiKey) return { text: "", error: "provider has no API key or OAuth token" }; + authHeader = `Bearer ${apiKey}`; + } + + const content: unknown[] = [ + { type: "text", text: contextText || "Describe this image." }, + toChatImagePart(imageUrl, detail), + ]; + const body = { + model: settings.model, + messages: [{ role: "user", content }], + stream: true, + max_tokens: 1024, + }; + + const headers: Record = { + "Content-Type": "application/json", + Authorization: authHeader, + ...(provider.headers ?? {}), + }; + + const linkedSignal = signalWithTimeout(settings.timeoutMs ?? 30_000, abortSignal); + const sidecarExit = sidecarEnter("vision"); + const t0 = Date.now(); + try { + const baseUrl = provider.baseUrl.replace(/\/$/, ""); + const url = new URL(`${baseUrl}/chat/completions`); + const res = await fetch(url.toString(), { + method: "POST", + headers, + body: JSON.stringify(body), + signal: linkedSignal.signal, + }); + recordOutcome?.(res.status); + if (!res.ok) { + const t = await res.text().catch(() => ""); + console.warn(`[vision] chat sidecar HTTP ${res.status} (${Date.now() - t0}ms)`); + return { text: "", error: `chat sidecar HTTP ${res.status}: ${redactSecretString(t.slice(0, 200))}` }; + } + if (!res.body) return { text: "", error: "chat sidecar returned no response body" }; + const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal); + let text = ""; + try { + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + const consumeLine = (line: string) => { + const s = line.replace(/^data:\s*/, "").trim(); + if (!s || s === "[DONE]") return; + try { + const j = JSON.parse(s); + const delta = j.choices?.[0]?.delta?.content; + if (typeof delta === "string") text += delta; + else if (Array.isArray(delta)) { + for (const part of delta) if (typeof part?.text === "string") text += part.text; + } + const message = j.choices?.[0]?.message?.content; + if (typeof message === "string") text += message; + } catch { /* ignore non-JSON data lines */ } + }; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + const lines = buf.split(/\r?\n/); + buf = lines.pop() ?? ""; + for (const line of lines) consumeLine(line); + } + consumeLine(buf); + const trimmed = text.trim(); + if (!trimmed) return { text: "", error: "chat sidecar produced no description" }; + return { text: trimmed }; + } finally { + detachBodyGuard(); + } + } catch (e) { + recordOutcome?.(e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error"); + console.warn(`[vision] chat sidecar error (${Date.now() - t0}ms)`); + return { text: "", error: e instanceof Error ? e.message : String(e) }; + } finally { + sidecarExit(); + linkedSignal.cleanup(); + } +} diff --git a/src/vision/index.ts b/src/vision/index.ts index 37dacc5962..fcdf763e98 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -3,6 +3,7 @@ import type { OcxConfig, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProvid import { modelInList } from "../types"; import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe"; import { describeImageAnthropic } from "./anthropic-describe"; +import { describeImageChat } from "./describe-chat"; import type { CodexAuthContext } from "../codex/auth-context"; import { getAccountSet } from "../oauth/store"; import type { ResolvedOpenAiForwardSidecar } from "../providers/openai-sidecar"; @@ -176,11 +177,54 @@ export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionP return undefined; } +/** Find the chat-adapter or google-adapter provider for `model` with usable auth. */ +function findChatVisionProvider(config: OcxConfig, model: string): { provider: OcxProviderConfig; providerName: string; model: string } | undefined { + let providerName = ""; + let bareModel = model; + if (model.includes("/")) { + const sep = model.indexOf("/"); + providerName = model.slice(0, sep); + bareModel = model.slice(sep + 1); + } + const isChatLike = (p: OcxProviderConfig) => p.adapter === "openai-chat" || p.adapter === "google"; + const hasAuth = (p: OcxProviderConfig) => { + if (p.apiKey ?? p.apiKeyPool?.[0]?.key) return true; + if (p.authMode === "oauth") return true; + return false; + }; + if (providerName) { + const provider = config.providers[providerName]; + if (provider && provider.disabled !== true && isChatLike(provider) && hasAuth(provider)) { + return { provider, providerName, model: bareModel }; + } + return undefined; + } + for (const [name, provider] of Object.entries(config.providers)) { + if (provider.disabled === true) continue; + if (!isChatLike(provider)) continue; + if (!hasAuth(provider)) continue; + const models = provider.models ?? []; + const defaultModel = provider.defaultModel; + if (bareModel === defaultModel || models.includes(bareModel) || models.some(m => m.endsWith("/" + bareModel) || m === bareModel)) { + return { provider, providerName: name, model: bareModel }; + } + } + for (const [name, provider] of Object.entries(config.providers)) { + if (provider.disabled === true) continue; + if (!isChatLike(provider)) continue; + if (!hasAuth(provider)) continue; + if (provider.liveModels === true) { + return { provider, providerName: name, model: bareModel }; + } + } + return undefined; +} + export function resolveVisionBackend( - explicit: "openai" | "anthropic" | undefined, + explicit: "openai" | "anthropic" | "chat" | undefined, anthropicSidecar: AnthropicVisionProvider | undefined, -): "openai" | "anthropic" { - if (explicit === "openai" || explicit === "anthropic") return explicit; +): "openai" | "anthropic" | "chat" { + if (explicit === "openai" || explicit === "anthropic" || explicit === "chat") return explicit; return anthropicSidecar ? "anthropic" : "openai"; } @@ -212,9 +256,10 @@ export function shouldResolveOpenAiVisionSidecar( } export interface VisionPlan { - backend: "openai" | "anthropic"; + backend: "openai" | "anthropic" | "chat"; forwardSidecar?: ResolvedOpenAiForwardSidecar; anthropicSidecar?: AnthropicVisionProvider; + chatSidecar?: { provider: OcxProviderConfig; providerName: string; model: string }; settings: VisionSettings; maxDescriptionsPerTurn: number; } @@ -240,6 +285,18 @@ export function planVisionSidecar( const backend = resolveVisionBackend(cfg.backend, anthropicSidecar); const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn); + if (backend === "chat") { + const model = cfg.model ?? ""; + const chatProvider = findChatVisionProvider(config, model); + if (!chatProvider) return undefined; + return { + backend, + chatSidecar: { provider: chatProvider.provider, providerName: chatProvider.providerName, model: chatProvider.model }, + settings: { model: resolveOpenAiVisionModel(config), timeoutMs: cfg.timeoutMs ?? DEFAULT_TIMEOUT_MS }, + maxDescriptionsPerTurn, + }; + } + if (backend === "anthropic") { if (!anthropicSidecar) return undefined; return { @@ -325,6 +382,20 @@ async function executeDescription( abortSignal, ); } + if (plan.backend === "chat") { + const sidecar = plan.chatSidecar; + if (!sidecar) return { text: "", error: "chat vision sidecar is unavailable" }; + return describeImageChat( + job.imageUrl, + job.detail, + job.contextText, + sidecar.provider, + sidecar.providerName, + { model: sidecar.model, timeoutMs: plan.settings.timeoutMs, detail: job.detail }, + abortSignal, + recordSidecarOutcome, + ); + } if (!plan.forwardSidecar) return { text: "", error: "OpenAI vision sidecar is unavailable" }; return describeImage( job.imageUrl, diff --git a/tests/vision-chat.test.ts b/tests/vision-chat.test.ts new file mode 100644 index 0000000000..1102d39e80 --- /dev/null +++ b/tests/vision-chat.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { describeImageChat } from "../src/vision/describe-chat"; +import type { OcxProviderConfig } from "../src/types"; + +const originalFetch = globalThis.fetch; +const image = "data:image/png;base64,aGVsbG8="; +const settings = { model: "vision-test", timeoutMs: 5000 }; + +afterEach(() => { globalThis.fetch = originalFetch; }); + +function chatSse(text: string): Response { + const body = [ + `data: ${JSON.stringify({ choices: [{ delta: { content: text } }] })}`, + "", + "data: [DONE]", + "", + ].join("\n"); + return new Response(body, { headers: { "content-type": "text/event-stream" } }); +} + +function geminiSse(text: string): Response { + const body = [ + `data: ${JSON.stringify({ candidates: [{ content: { parts: [{ text }] } }] })}`, + "", + `data: ${JSON.stringify({ candidates: [{ finishReason: "STOP" }] })}`, + "", + ].join("\n"); + return new Response(body, { headers: { "content-type": "text/event-stream" } }); +} + +describe("chat vision sidecar", () => { + test("sends an image_url through an OpenAI-compatible provider", async () => { + let capturedUrl = ""; + let capturedBody: Record | undefined; + globalThis.fetch = (async (url, init) => { + capturedUrl = String(url); + capturedBody = JSON.parse(String(init?.body)); + return chatSse("Mimo description"); + }) as typeof fetch; + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://vision.example/v1", + authMode: "key", + apiKey: "test-key", + }; + + const result = await describeImageChat(image, "high", "describe this", provider, "mimo", settings); + + expect(result).toEqual({ text: "Mimo description" }); + expect(capturedUrl).toBe("https://vision.example/v1/chat/completions"); + expect(capturedBody?.model).toBe("vision-test"); + expect(capturedBody?.messages[0].content).toEqual([ + { type: "text", text: "describe this" }, + { type: "image_url", image_url: { url: image, detail: "high" } }, + ]); + }); + + test("uses the native Google adapter wire format", async () => { + let capturedUrl = ""; + let capturedBody: Record | undefined; + globalThis.fetch = (async (url, init) => { + capturedUrl = String(url); + capturedBody = JSON.parse(String(init?.body)); + return geminiSse("Gemini description"); + }) as typeof fetch; + const provider: OcxProviderConfig = { + adapter: "google", + baseUrl: "https://generativelanguage.googleapis.com", + authMode: "key", + apiKey: "test-key", + }; + + const result = await describeImageChat(image, "high", "describe this", provider, "gemini", { model: "gemini-test", timeoutMs: 5000 }); + + expect(result).toEqual({ text: "Gemini description" }); + expect(capturedUrl).toBe("https://generativelanguage.googleapis.com/v1beta/models/gemini-test:streamGenerateContent?alt=sse"); + expect(capturedBody?.contents?.[0]?.parts).toEqual([ + { text: "describe this" }, + { inline_data: { mime_type: "image/png", data: "aGVsbG8=" } }, + ]); + }); +});