Skip to content

feat(vision): add chat and Google sidecars #1161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
waw4303 wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft

feat(vision): add chat and Google sidecars #1161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions gui/src/i18n/de.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ export const de: Record<TKey, string> = {
"dash.backendAuto": "Automatisch",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "Sidecar-Einstellungen gespeichert. Angewendet bei der nächsten Anfrage.",
"dash.sidecarSaveFailed": "Sidecar-Einstellungen konnten nicht gespeichert werden.",
"dash.injectionLabel": "Sub-Agent-Delegation",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/en.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ export const en = {
"dash.backendAuto": "Auto",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "Sidecar settings saved. Applied on the next request.",
"dash.sidecarSaveFailed": "Failed to save sidecar settings.",
"dash.injectionLabel": "Sub-agent delegation",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ja.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ export const ja: Record<TKey, string> = {
"dash.backendAuto": "自動",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "サイドカー設定を保存しました。次回リクエスト時に適用されます。",
"dash.sidecarSaveFailed": "サイドカー設定の保存に失敗しました。",
"dash.injectionLabel": "サブエージェント委任",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ko.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ export const ko: Record<TKey, string> = {
"dash.backendAuto": "자동",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "사이드카 설정이 저장됐습니다. 다음 요청부터 적용됩니다.",
"dash.sidecarSaveFailed": "사이드카 설정 저장에 실패했습니다.",
"dash.injectionLabel": "서브에이전트 위임",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ru.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ export const ru: Record<TKey, string> = {
"dash.backendAuto": "Авто",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "Настройки сайдкара сохранены. Вступят в силу со следующего запроса.",
"dash.sidecarSaveFailed": "Не удалось сохранить настройки сайдкара.",
"dash.injectionLabel": "Делегирование подагентам",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/zh.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ export const zh: Record<TKey, string> = {
"dash.backendAuto": "自动",
"dash.backendOpenAI": "OpenAI",
"dash.backendAnthropic": "Anthropic",
"dash.backendChat": "Chat API",
"dash.sidecarSaved": "附属设置已保存。将在下一个请求时生效。",
"dash.sidecarSaveFailed": "保存附属设置失败。",
"dash.injectionLabel": "子代理委托",
Expand Down
1 change: 1 addition & 0 deletions gui/src/pages/claude-code-sections.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ export function ClaudeCodeSettingsCard({
{ value: "auto", label: t("dash.backendAuto") },
{ value: "openai", label: t("dash.backendOpenAI") },
{ value: "anthropic", label: t("dash.backendAnthropic") },
...(key === "visionSidecar" ? [{ value: "chat", label: t("dash.backendChat") }] : []),
]}
onChange={value => {
// Auto may exist as an empty in-memory draft so the model input
Expand Down
2 changes: 1 addition & 1 deletion gui/src/pages/claude-manual-env.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* copy-paste shell block is directly unit-testable (tests/claude-manual-env.test.ts).
*/

export type SidecarBackend = "openai" | "anthropic";
export type SidecarBackend = "openai" | "anthropic" | "chat";
export interface SidecarOverride { backend?: SidecarBackend; model?: string }

export interface ClaudeManualEnvState {
Expand Down
30 changes: 28 additions & 2 deletions gui/src/pages/dashboard-overview-sections.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ export function DashboardMaintenancePanel({ d }: { d: Dash }) {
export function DashboardSidecarPanels({ d }: { d: Dash }) {
const {
t, settings, settingsSaving, toggleCodexAutoStart,
sidecar, sidecarSaving, sidecarModels, models, saveSidecar,
sidecar, sidecarSaving, sidecarModels, visionSidecarModels, models, saveSidecar,
shadowCall, shadowCallSaving, shadowCallHelpTriggerRef, shadowCallHelpOpen, setShadowCallHelpOpen, saveShadowCall,
} = d;

Expand Down Expand Up @@ -317,9 +317,35 @@ export function DashboardSidecarPanels({ d }: { d: Dash }) {
<div className="panel dash-sidecar-card" aria-busy={!sidecar || undefined}>
<div className="dash-sidecar-card__row">
<div className="font-semibold">{t("dash.visionSidecar")}</div>
<Select
value={sidecar?.vision.backend ?? "auto"}
options={[
{ value: "auto", label: t("dash.backendAuto") },
{ value: "openai", label: t("dash.backendOpenAI") },
{ value: "anthropic", label: t("dash.backendAnthropic") },
{ value: "chat", label: t("dash.backendChat") },
]}
onChange={backend => {
const target = backend === "auto" ? null : backend as "openai" | "anthropic" | "chat";
const current = sidecar?.vision.model ?? "";
const currentBackend = sidecarBackendForModel(models, current);
const currentOption = visionSidecarModels.find(option => option.value === current);
const compatible = target === null
? visionSidecarModels.find(option => sidecarBackendForModel(models, option.value) === "openai")
Comment on lines +333 to +334

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 6, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Do not save an OpenAI model for Auto vision

When users switch the vision backend to Auto on a setup with an Anthropic OAuth sidecar, this branch clears backend but persists the first OpenAI model. Runtime auto resolution still prefers Anthropic when available, so the next image request asks the Anthropic sidecar to run an OpenAI model id and the description fails. Auto should preserve a backend-compatible model or clear model so the runtime can use its backend default.

AGENTS.md reference: gui/AGENTS.md:L9-L10

Useful? React with 👍 / 👎.

: currentOption && currentBackend === target
? currentOption
: visionSidecarModels.find(option => sidecarBackendForModel(models, option.value) === target);
void saveSidecar({ vision: {
backend: target,
...(compatible?.value ? { model: compatible.value } : {}),
} });
}}
disabled={!sidecar || sidecarSaving}
label={t("dash.sidecarBackend")}
/>
<Select
value={sidecar?.vision.model ?? "gpt-5.6-luna"}
options={sidecarModels}
options={visionSidecarModels}
onChange={model => { void saveSidecar({ vision: { model, backend: sidecarBackendForModel(models, model) } }); }}
disabled={!sidecar || sidecarSaving}
label={t("dash.sidecarModel")}
Expand Down
23 changes: 20 additions & 3 deletions gui/src/pages/dashboard-shared.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ export async function requireJson<T>(res: Response, fallbackMessage?: string): P

export interface HealthData { status: string; version: string; uptime: number }
export interface ProviderInfo { name: string; adapter: string; baseUrl: string; defaultModel?: string; hasApiKey: boolean }
export interface ModelInfo { id: string; provider: string; namespaced: string; owned_by?: string }
export interface ModelInfo { id: string; provider: string; namespaced: string; owned_by?: string; disabled?: boolean }
export interface SettingsData {
codexAutoStart: boolean;
port: number;
Expand All @@ -54,7 +54,7 @@ export interface SettingsData {
diagnosticStale: boolean;
};
}
export type SidecarBackend = "openai" | "anthropic";
export type SidecarBackend = "openai" | "anthropic" | "chat";
export interface SidecarSetting { backend?: SidecarBackend; model: string }
export interface SidecarData { webSearch: SidecarSetting; vision: SidecarSetting }
export interface SidecarPatch {
Expand Down Expand Up @@ -161,6 +161,20 @@ export function sidecarModelOptions(models: ModelInfo[]) {
return out;
}

/** Vision can use enabled routed providers; namespaced values preserve provider selection. */
export function visionSidecarModelOptions(models: ModelInfo[]) {
const out: Array<{ value: string; label: string }> = [];
for (const model of models) {
if (model.disabled === true) continue;
if (model.provider === "openai" || model.provider === "anthropic") {
out.push({ value: model.id, label: `${model.provider}/${model.id}` });
} else {
out.push({ value: model.namespaced, label: model.namespaced });

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 6, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Filter the vision picker to usable chat providers

This branch adds every non-OpenAI/Anthropic row to the vision picker and sidecarBackendForModel classifies it as chat, but the runtime chat sidecar only accepts providers whose adapter is openai-chat or google. Selecting rows such as anthropic-apikey/..., combos, or other adapters saves a model the backend can never plan, so text-only requests strip the image as unavailable. Source this list from runtime-eligible providers or include adapter capability data.

AGENTS.md reference: gui/AGENTS.md:L9-L10

Useful? React with 👍 / 👎.

}
}
return out;
}

/** Options for shadow-call replacement models use the proxy's canonical routing id. */
export function shadowCallModelOptions(models: ModelInfo[], current: string | undefined) {
const out = [{ value: "", label: "—" }, ...models.map(model => ({ value: model.namespaced, label: model.namespaced }))];
Expand All @@ -169,7 +183,10 @@ export function shadowCallModelOptions(models: ModelInfo[], current: string | un
}

export function sidecarBackendForModel(models: ModelInfo[], modelId: string): SidecarBackend {
return models.find(model => model.id === modelId)?.provider === "anthropic" ? "anthropic" : "openai";
const model = models.find(item => item.id === modelId || item.namespaced === modelId);
if (model?.provider === "anthropic") return "anthropic";
if (model?.provider === "openai") return "openai";
return "chat";
Comment on lines 185 to +189

@coderabbitai coderabbitai Bot Aug 6, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | 🏗️ Heavy lift

Do not classify an unresolved model ID as the chat backend.

A missing configured vision model is retained in gui/src/pages/use-dashboard-data.ts. The current helper maps that unresolved ID to chat. Selecting Chat API then preserves the stale ID instead of selecting a discovered, namespaced Chat API model. The backend cannot resolve that stale model to a configured provider.

  • gui/src/pages/dashboard-shared.ts#L185-L189: return a distinct unresolved result instead of using chat as the fallback.
  • gui/src/pages/dashboard-overview-sections.tsx#L330-L337: retain the current model only when it resolves to an enabled model with the selected backend.

As per path instructions, GUI state changes must stay consistent with management API responses.

📍 Affects 2 files
  • gui/src/pages/dashboard-shared.ts#L185-L189 (this comment)
  • gui/src/pages/dashboard-overview-sections.tsx#L330-L337
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@gui/src/pages/dashboard-shared.ts` around lines 185 - 189, The unresolved
model fallback in sidecarBackendForModel must return a distinct unresolved
result rather than "chat"; update gui/src/pages/dashboard-shared.ts lines
185-189 accordingly. In gui/src/pages/dashboard-overview-sections.tsx lines
330-337, retain the current model only when it resolves to an enabled model
whose backend matches the selected backend; otherwise select a discovered
namespaced model consistent with the management API response.

Source: Path instructions

}

let lastInputWasKeyboard = false;
Expand Down
18 changes: 13 additions & 5 deletions gui/src/pages/use-dashboard-data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import {
readDashboardSectionFromHash,
requireJson,
sidecarModelOptions,
visionSidecarModelOptions,
useModalDialog,
} from "./dashboard-shared";

Expand Down Expand Up @@ -453,13 +454,20 @@ export function useDashboardData(apiBase: string) {
}, [grouped, modelQuery]);
const sidecarModels = useMemo(() => {
const opts = sidecarModelOptions(models);
for (const id of [sidecar?.webSearch.model, sidecar?.vision.model]) {
if (id && !opts.some(option => option.value === id)) {
opts.unshift({ value: id, label: id });
}
const current = sidecar?.webSearch.model;
const currentModel = models.find(model => model.id === current || model.namespaced === current);
if (current && (currentModel?.provider === "openai" || currentModel?.provider === "anthropic")
&& !opts.some(option => option.value === current)) {
opts.unshift({ value: current, label: current });
}
return opts;
}, [models, sidecar]);
const visionSidecarModels = useMemo(() => {
const opts = visionSidecarModelOptions(models);
const current = sidecar?.vision.model;
if (current && !opts.some(option => option.value === current)) opts.unshift({ value: current, label: current });
return opts;
}, [models, sidecar]);

const saveSidecar = async (patch: SidecarPatch) => {
if (!sidecar || sidecarSaving) return;
Expand Down Expand Up @@ -745,7 +753,7 @@ export function useDashboardData(apiBase: string) {
updateCheck, updateError, updateJob, reconnecting, error,
effortCapHelpTriggerRef, updateTriggerRef, maHelpTriggerRef, shadowCallHelpTriggerRef,
effortCapHelpDialogRef, updateDialogRef, maHelpDialogRef, shadowCallHelpDialogRef,
filteredGroups, sidecarModels,
filteredGroups, sidecarModels, visionSidecarModels,
saveSidecar, saveShadowCall, switchMaMode, toggleCodexAutoStart, runSync, clearSyncFeedback,
fetchUpdateCheck, closeUpdateDialog, openUpdateDialog, changeUpdateChannel, runUpdate,
};
Expand Down
2 changes: 1 addition & 1 deletion src/cli/agent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const USAGE = `Usage:
ocx agent effort <status|set> [--main <level|->] [--subagent <level|->] [--json]
ocx agent subagents <status|set|clear> [model,model...] [--json]
ocx agent fallback <status|set|clear> [model,model...] [--poll-ms <5000-600000>] [--json]
ocx agent sidecar <status|web|vision> [--model <id|->] [--backend <openai|anthropic|->]
ocx agent sidecar <status|web|vision> [--model <id|->] [--backend <openai|anthropic|chat|->]

@coderabbitai coderabbitai Bot Aug 6, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

Separate the web and vision backend choices in the usage text.

Line 23 advertises chat for ocx agent sidecar web. The management API rejects webSearch.backend: "chat". Document openai|anthropic for web and openai|anthropic|chat for vision.

Proposed fix 
-  ocx agent sidecar <status|web|vision> [--model <id|->] [--backend <openai|anthropic|chat|->]
+  ocx agent sidecar web [--model <id|->] [--backend <openai|anthropic|->]
+  ocx agent sidecar vision [--model <id|->] [--backend <openai|anthropic|chat|->]
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
ocx agent sidecar <status|web|vision> [--model <id|->] [--backend <openai|anthropic|chat|->]
ocx agent sidecar web [--model <id|->] [--backend <openai|anthropic|->]
ocx agent sidecar vision [--model <id|->] [--backend <openai|anthropic|chat|->]
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/cli/agent.ts` at line 23, Update the sidecar usage text in the agent CLI
help to document backend choices by status: show openai|anthropic for web and
openai|anthropic|chat for vision, while preserving the existing model and status
syntax.

[--reasoning <level>] [--max-descriptions <n>] [--json]`;

function clearable(value: string | undefined): string | null | undefined {
Expand Down
2 changes: 1 addition & 1 deletion src/cli/integrations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const CLAUDE_USAGE = `Usage:
[--compact-window <tokens|default>] [--inject-agents <on|off>]
[--small-fast-model <id|->] [--model-map <from=to,from=to|->]
[--blocked-skills <name,name|->] [--web-model <id|->] [--web-backend <openai|anthropic|->]
[--vision-model <id|->] [--vision-backend <openai|anthropic|->] [--json]`;
[--vision-model <id|->] [--vision-backend <openai|anthropic|chat|->] [--json]`;

const GROK_USAGE = `Usage:
ocx grok [status] [--json]
Expand Down
9 changes: 5 additions & 4 deletions src/server/management/agent-settings-routes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1004,8 +1004,9 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise
if (section === undefined || section === null) continue;
if (!isPlainObject(section)) return jsonResponse({ error: `${field} must be an object or null` }, 400);
if (section.backend !== undefined && section.backend !== null
&& section.backend !== "openai" && section.backend !== "anthropic") {
return jsonResponse({ error: `${field}.backend must be openai, anthropic, or null` }, 400);
&& section.backend !== "openai" && section.backend !== "anthropic" && (field !== "visionSidecar" || section.backend !== "chat")) {
const accepted = field === "visionSidecar" ? "openai, anthropic, chat, or null" : "openai, anthropic, or null";
return jsonResponse({ error: `${field}.backend must be ${accepted}` }, 400);
}
if (section.model !== undefined && typeof section.model !== "string") {
return jsonResponse({ error: `${field}.model must be a string` }, 400);
Expand All @@ -1019,8 +1020,8 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise
delete next[field];
continue;
}
const requested = section as { backend?: "openai" | "anthropic" | null; model?: string };
const override: NonNullable<OcxClaudeCodeConfig[typeof field]> = { ...next[field] };
const requested = section as { backend?: "openai" | "anthropic" | "chat" | null; model?: string };
const override = { ...(next[field] as Record<string, unknown> | undefined) } as Record<string, unknown>;
if (requested.backend === null) delete override.backend;
else if (requested.backend !== undefined) override.backend = requested.backend;
if (requested.model === "") delete override.model;
Expand Down
6 changes: 3 additions & 3 deletions src/server/management/config-routes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,8 +339,8 @@ export async function handleConfigRoutes(ctx: ManagementContext): Promise<Respon
return jsonResponse({ error: "webSearch.backend must be openai, anthropic, or null" }, 400);
}
if (body.vision && body.vision.backend !== undefined
&& body.vision.backend !== null && body.vision.backend !== "openai" && body.vision.backend !== "anthropic") {
return jsonResponse({ error: "vision.backend must be openai, anthropic, or null" }, 400);
&& body.vision.backend !== null && body.vision.backend !== "openai" && body.vision.backend !== "anthropic" && body.vision.backend !== "chat") {
return jsonResponse({ error: "vision.backend must be openai, anthropic, chat, or null" }, 400);
}
if (body.vision && body.vision.maxDescriptionsPerTurn !== undefined
&& (typeof body.vision.maxDescriptionsPerTurn !== "number"
Expand All @@ -367,7 +367,7 @@ export async function handleConfigRoutes(ctx: ManagementContext): Promise<Respon
else config.visionSidecar.model = body.vision.model;
}
if (body.vision.backend === null) delete config.visionSidecar.backend;
else if (body.vision.backend === "openai" || body.vision.backend === "anthropic") {
else if (body.vision.backend === "openai" || body.vision.backend === "anthropic" || body.vision.backend === "chat") {
config.visionSidecar.backend = body.vision.backend;
}
if (typeof body.vision.maxDescriptionsPerTurn === "number") {
Expand Down
4 changes: 2 additions & 2 deletions src/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ export interface OcxClaudeCodeConfig {
/** Claude-originated web-search override. Unset fields inherit the global sidecar settings. */
webSearchSidecar?: { backend?: "openai" | "anthropic"; model?: string };
/** Claude-originated vision override. Unset fields inherit the global sidecar settings. */
visionSidecar?: { backend?: "openai" | "anthropic"; model?: string };
visionSidecar?: { backend?: "openai" | "anthropic" | "chat"; model?: string };
/** Persisted Claude Desktop four-family routing profile. */
desktopProfile?: OcxClaudeDesktopProfile;
/** Auto-reconcile Desktop 3P config when provider catalog changes. Default: enabled. */
Expand Down Expand Up @@ -980,7 +980,7 @@ export interface OcxVisionSidecarConfig {
/** Master switch. Default: enabled when the selected backend has a usable credential. */
enabled?: boolean;
/** Description backend. Unset prefers a usable stored Anthropic OAuth credential, else OpenAI. */
backend?: "openai" | "anthropic";
backend?: "openai" | "anthropic" | "chat";
/** Vision model that describes images. */
model?: string;
/** Max description cache misses admitted in one main-model turn. Zero disables description calls. */
Expand Down
Loading
Loading