diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index f75b7d26d..537d483d4 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -48,8 +48,8 @@ other; closing one is a maintainer decision and neither is stale. | RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | MERGED | | RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `1f07c00b8` | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | MERGED | | RI-08 | `feat/ri-08-cost-aware-routing` | `dev` (post-#1014 merge) | `410db97e4` | #1015 | https://github.com/lidge-jun/opencodex/pull/1015 | MERGED | -| RI-09 | `feat/ri-09-route-explainability-api` | `dev` (post-#1015 merge) | `d887c1202` | #1016 | https://github.com/lidge-jun/opencodex/pull/1016 | rebased / review in progress | -| RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued | +| RI-09 | `feat/ri-09-route-explainability-api` | `dev` (post-#1015 merge) | `68d3aa083` | #1016 | https://github.com/lidge-jun/opencodex/pull/1016 | MERGED | +| RI-10 | `feat/ri-10-routing-intelligence-ui` | `dev` (post-#1016 merge; rebasing) | pending | #1018 | https://github.com/lidge-jun/opencodex/pull/1018 | OPEN | ## Per-PR acceptance log @@ -270,7 +270,7 @@ other; closing one is a maintainer decision and neither is stale. (5) assembleCandidateEvidence typed as `OcxConfig` after the RI-08 health/ quota/cost merge. - Final commit: `d887c120282c8d381f92cd11c1eebe2373a27281` -- PR: #1016 https://github.com/lidge-jun/opencodex/pull/1016 +- PR: #1016 https://github.com/lidge-jun/opencodex/pull/1016 (MERGED) - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) - `bun run test tests/route-explainability.test.ts`: 10/10 pass - @@ -282,6 +282,39 @@ other; closing one is a maintainer decision and neither is stale. - `bun run privacy:scan`: passed - Remaining Low findings: none +### RI-10 - feat/ri-10-routing-intelligence-ui + +- Base SHA: `68d3aa0836648ae1d7b592fc5aa3b30146c0886d` (`dev` after #1016 merge) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 4 fixed pre-push - + 1. GUI lint: hardcoded "profiles" in an error literal (i18n rule) - now a + key-less status code; + 2. GUI lint: setState-in-effect for the initial load - deferred via + setTimeout(0); + 3. missing `.checkbox` CSS class - added to styles.css; dry-run/analytics + tables reuse the existing `.tbl` grammar; + 4. dev-mode GUI session bootstrap cannot authenticate through the Vite + proxy - the screenshot is captured same-origin against the production + GUI served by the backend instead. +- Final commit: pending (recorded after commit) +- PR: #1018 https://github.com/lidge-jun/opencodex/pull/1018 +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run lint:gui`: PASSED (0 errors) + - `bun run build:gui`: PASSED (production build + prepare:package) + - `bun run test` (12 routing suites): 258/258 pass + - `bun run privacy:scan`: passed + - docs-site `bun run build`: 216 pages built, PASSED + - Locale parity: compile-checked TKey set (all six locales updated) + - Screenshot: live same-origin capture of `#routing` (profiles + dry-run + + analytics) against a temporary config; uploaded to the PR via comment + attachment +- Environment note: the temporary screenshot backend briefly rewrote the + Codex/Grok fence to port 10200; restored with `ocx ensure` to the live + proxy (10100) and verified. Test processes and temp files cleaned up. +- Remaining Low findings: none +- Remaining Low findings: none + ## Baseline note The full-suite baseline on this Windows machine did not complete within the diff --git a/docs-site/public/pr-screenshots/1018-routing-intelligence.png b/docs-site/public/pr-screenshots/1018-routing-intelligence.png new file mode 100644 index 000000000..4325805fe Binary files /dev/null and b/docs-site/public/pr-screenshots/1018-routing-intelligence.png differ diff --git a/docs-site/src/content/docs/ja/reference/configuration/routing.md b/docs-site/src/content/docs/ja/reference/configuration/routing.md index 2c4c32092..2eda8fa3c 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/routing.md +++ b/docs-site/src/content/docs/ja/reference/configuration/routing.md @@ -84,3 +84,26 @@ selector の後には bare native OpenAI-family id だけを指定できます - 空ではない `inputModalities` 交差。省略されたメンバー値を `["text"]` として扱います。 コンテキスト メタデータのない裸のリレー ID、または接続されていないモダリティを持つターゲットは、カタログからコンボを削除します。同期によって概要の警告が表示され、ダッシュボードで **注意が必要** とマークされます。コンテキスト メタデータを追加し、モダリティを調整したり、検出可能な互換性のある機能を備えたターゲット モデルを追加したりできます。 + +## ルーティングポリシープロファイル（`config.routingProfiles`） + +明示的に要求された `policy/`（または設定されたエイリアス）が、固定された候補許可リストの中から、ハードな能力要件と決定的で説明可能なスコアリングで選択します。既存のモデル ID が暗黙的にプロファイルを通ることはありません。`candidates`（明示的な許可リスト）、オプションの `alias`、`require`（`minContextWindow`、`minQuotaHeadroom`、`tools`、`imageInput`、`structuredOutput`、`localOnly`、`remoteAllowed`、`encryptedCodexTasks`、`reasoningEffort`、`serviceTier`）、`optimize`（latency/health/cost/quota の重み）、`limits.maxEstimatedCostUsd`、`unknownEvidence`（allow/penalize/exclude）をサポートします。未知はゼロや無料にはなりません。 + +CLI: `ocx route policy list`、`ocx route policy show `、`ocx route policy dry-run --model-context --tools`、`ocx route policy evaluate `。 + +コンボは明示的な順序・重み付きターゲットのルーティングとフェイルオーバーです。ポリシープロファイルは、候補間の証拠に基づく選択です。 + +## リクエスト履歴とルーティング分析 + +- `GET /api/request-history` - 派生インデックス（`routing-history.sqlite`）からのカーソルページング付き全履歴。フィルタ: `provider`、`model`、`requestedModel`、`status`、`conversationId`、`surface`、`inboundProtocol`、`apiKeyId`、`profileId`、`fallback`、`from`、`to`。 +- `GET /api/request-history/:requestId/route-decision` - このルートが選ばれた理由（トレース、候補、除外、スコア、プロファイル+リビジョン、実行試行、結果）。 +- `GET /api/routing-analytics` - 成功率・失敗率・フォールバック率、p50/p95/p99 の所要時間と TTFT、不完全ストリーム率、クールダウン失敗数、成功あたりの推定コスト、カバレッジ、信頼度、切り捨てフラグ。 +- `GET /api/routing-profiles`、`POST /api/routing-profiles/dry-run` - プロファイル参照とドライラン評価（上流への送信なし）。 + +返される履歴とルート決定ペイロードは、マスク済みのリクエストメタデータのみを公開します（例: 不透明な `apiKeyId` ラベル）。資格情報、生のプロンプト本文、プロバイダのシークレットは含みません。 + +CLI: `ocx logs explain `、`ocx logs rebuild-index`、`ocx logs index-status`。 + +## 移行 + +`routingProfiles` は任意の追加設定です。既存の設定ファイルと古い `usage.jsonl` 行はそのまま読み込めます。インデックスは使い捨てで、削除すると次回クエリ時に `usage.jsonl` から自動再構築されます。自動チューニングは行われません。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/routing.md b/docs-site/src/content/docs/ko/reference/configuration/routing.md index 0ae010707..4615b6362 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/routing.md +++ b/docs-site/src/content/docs/ko/reference/configuration/routing.md @@ -82,3 +82,26 @@ combo는 목록에 오를 수 없더라도 계속 직접 라우팅할 수 있습 - 비어 있지 않은 `inputModalities` 교집합. 생략된 member value는 `["text"]`로 취급합니다. context metadata가 없는 bare relay id이거나 modalities가 서로 겹치지 않는 target이 있으면 combo가 catalog에서 빠집니다. sync는 summary warning을 내고 dashboard는 이를 **Needs attention**으로 표시합니다. context metadata를 추가하거나, modalities를 맞추거나, 발견 가능한 호환 capability를 가진 model을 대상으로 삼으십시오. + +## 라우팅 정책 프로필 (`config.routingProfiles`) + +명시적으로 요청된 `policy/`(또는 설정된 별칭)가 고정된 후보 허용 목록에서 하드 능력 요구사항과 결정적·설명 가능한 점수로 선택합니다. 기존 모델 ID가 암시적으로 프로필을 통과하지 않습니다. `candidates`(명시적 허용 목록), 선택적 `alias`, `require`(`minContextWindow`, `minQuotaHeadroom`, `tools`, `imageInput`, `structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`, `reasoningEffort`, `serviceTier`), `optimize`(latency/health/cost/quota 가중치), `limits.maxEstimatedCostUsd`, `unknownEvidence`(allow/penalize/exclude)를 지원합니다. 알 수 없음은 0이나 무료가 되지 않습니다. + +CLI: `ocx route policy list`, `ocx route policy show `, `ocx route policy dry-run --model-context --tools`, `ocx route policy evaluate `. + +콤보는 명시적인 순서·가중치 대상 라우팅 및 장애 조치입니다. 정책 프로필은 후보 간 증거 기반 선택입니다. + +## 요청 기록 및 라우팅 분석 + +- `GET /api/request-history` - 파생 인덱스(`routing-history.sqlite`)에서 커서 페이지네이션으로 전체 기록을 조회. 필터: `provider`, `model`, `requestedModel`, `status`, `conversationId`, `surface`, `inboundProtocol`, `apiKeyId`, `profileId`, `fallback`, `from`, `to`. +- `GET /api/request-history/:requestId/route-decision` - 이 경로가 선택된 이유(추적, 후보, 제외, 점수, 프로필+리비전, 실행 시도, 결과). +- `GET /api/routing-analytics` - 성공/실패/취소/폴백 비율, p50/p95/p99 소요 시간 및 TTFT, 불완전 스트림 비율, 쿨다운 실패 수, 성공 요청당 추정 비용, 커버리지, 신뢰도, 잘림 플래그. +- `GET /api/routing-profiles`, `POST /api/routing-profiles/dry-run` - 프로필 조회와 드라이런 평가(업스트림 전송 없음). + +반환되는 히스토리와 라우트 결정 페이로드는 마스킹된 요청 메타데이터만 노출합니다(예: 불투명한 `apiKeyId` 라벨). 자격 증명, 원본 프롬프트 본문, 공급자 시크릿은 포함하지 않습니다. + +CLI: `ocx logs explain `, `ocx logs rebuild-index`, `ocx logs index-status`. + +## 마이그레이션 + +`routingProfiles`는 선택적 추가 설정입니다. 기존 설정 파일과 이전 `usage.jsonl` 행은 그대로 읽힙니다. 인덱스는 일회용이며 삭제 시 다음 쿼리에서 `usage.jsonl`로 자동 재구축됩니다. 자동 튜닝은 없습니다. diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 1c7b143f9..862dd9867 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -189,3 +189,36 @@ Codex picker list it only when every target exposes capabilities that can be int A bare relay id with no context metadata or targets with disjoint modalities removes the combo from the catalog. Sync emits a summary warning and the dashboard marks it **Needs attention**. Add context metadata, align modalities, or target models with discoverable compatible capabilities. + +## Request history and routing analytics + +- `GET /api/request-history` - cursor-paginated full history from the derived + index (`routing-history.sqlite`), with filters (`provider`, `model`, + `requestedModel`, `status`, `conversationId`, `surface`, `inboundProtocol`, + `apiKeyId`, `profileId`, `fallback`, `from`, `to`) and opaque `cursor` + pagination. `GET /api/request-history/:requestId` returns one canonical row. +- `GET /api/request-history/:requestId/route-decision` - the why-this-route + explanation: trace (candidates, exclusions, score components, profile + + revision), execution attempt sequence, and final outcome. +- `GET /api/routing-analytics` - success/failure/cancelled/fallback rates, + p50/p95/p99 duration and TTFT, incomplete-stream rate, cooldown-triggering + failures, cost per successful request, coverage, confidence, and an + explicit truncation flag. +- `GET /api/routing-profiles`, `POST /api/routing-profiles/dry-run` - profile + inspection and dry-run evaluation (no upstream dispatch). + +Returned history and route-decision payloads expose only masked request metadata +(for example opaque `apiKeyId` labels). They do not include credentials, raw +prompt bodies, or provider secrets. + +CLI: `ocx logs explain `, `ocx logs rebuild-index`, +`ocx logs index-status`, `ocx route policy list | show | dry-run | evaluate`. + +## Migration + +`routingProfiles` is optional and additive: existing config files load +unchanged. Old `usage.jsonl` rows without `routeDecision` parse unchanged. +The history index is disposable - deleting `routing-history.sqlite` triggers +an automatic rebuild from `usage.jsonl` on the next query; `ocx logs +rebuild-index` forces one. Nothing in this system auto-tunes weights, +budgets, or candidate sets. diff --git a/docs-site/src/content/docs/ru/reference/configuration/routing.md b/docs-site/src/content/docs/ru/reference/configuration/routing.md index e7deed4ae..429610c00 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/routing.md +++ b/docs-site/src/content/docs/ru/reference/configuration/routing.md @@ -99,3 +99,26 @@ Combo остаётся доступной для прямой маршрутиз из каталога. Sync выводит итоговое предупреждение, а дашборд помечает её как **Needs attention**. Добавьте метаданные контекста, согласуйте модальности или выберите модели с обнаруживаемыми совместимыми возможностями. + +## Профили маршрутизации (`config.routingProfiles`) + +Явно запрошенный `policy/` (или настроенный псевдоним) выбирает среди фиксированного разрешённого списка кандидатов по жёстким требованиям к возможностям и детерминированной объяснимой оценке. Существующие идентификаторы моделей никогда не проходят через профиль неявно. Поддерживаются: `candidates` (явный список), необязательный `alias`, `require` (`minContextWindow`, `minQuotaHeadroom`, `tools`, `imageInput`, `structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`, `reasoningEffort`, `serviceTier`), `optimize` (веса latency/health/cost/quota), `limits.maxEstimatedCostUsd`, `unknownEvidence` (allow/penalize/exclude). Неизвестное не становится нулём или бесплатным. + +CLI: `ocx route policy list`, `ocx route policy show `, `ocx route policy dry-run --model-context --tools`, `ocx route policy evaluate `. + +Комбо — это явная маршрутизация с порядком/весами и отказоустойчивостью. Профиль — это выбор на основе доказательств среди кандидатов. + +## История запросов и аналитика маршрутизации + +- `GET /api/request-history` - полная история с курсорной пагинацией из производного индекса (`routing-history.sqlite`). Фильтры: `provider`, `model`, `requestedModel`, `status`, `conversationId`, `surface`, `inboundProtocol`, `apiKeyId`, `profileId`, `fallback`, `from`, `to`. +- `GET /api/request-history/:requestId/route-decision` - объяснение выбора маршрута (трасса, кандидаты, исключения, оценки, профиль+ревизия, попытки, результат). +- `GET /api/routing-analytics` - доли успеха/отказа/отмены/фолбэка, p50/p95/p99 длительность и TTFT, доля неполных потоков, сбои кулдауна, оценка стоимости за успешный запрос, покрытие, доверие, флаг усечения. +- `GET /api/routing-profiles`, `POST /api/routing-profiles/dry-run` - просмотр профилей и пробная оценка (без отправки запросов). + +Возвращаемые записи истории и решений маршрута содержат только маскированные метаданные запроса (например, непрозрачные метки `apiKeyId`). Учётные данные, сырые тела промптов и секреты провайдеров не включаются. + +CLI: `ocx logs explain `, `ocx logs rebuild-index`, `ocx logs index-status`. + +## Миграция + +`routingProfiles` — необязательная аддитивная настройка. Существующие конфиги и старые строки `usage.jsonl` загружаются без изменений. Индекс одноразовый: при удалении он автоматически перестраивается из `usage.jsonl` при следующем запросе. Автонастройки нет. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md index 91de85c76..b718ed4ed 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md @@ -89,3 +89,26 @@ selector 校验、冲突规则和隐私说明见[提供方配置](/reference/con 如果是一个没有上下文元数据的裸 relay id，或者目标之间的模态互不相交，combo 就会从 目录中移除。同步时会输出一条汇总警告，仪表板会将其标记为 **Needs attention**。 补充上下文元数据、对齐模态，或者把目标模型切换为可发现且兼容的能力。 + +## 路由策略配置文件（`config.routingProfiles`） + +显式请求的 `policy/`（或配置的别名）会在固定的候选白名单中，根据硬性能力要求与确定性、可解释的评分进行选择。现有模型 ID 永远不会隐式经过配置文件。支持 `candidates`（显式白名单）、可选 `alias`、`require`（`minContextWindow`、`minQuotaHeadroom`、`tools`、`imageInput`、`structuredOutput`、`localOnly`、`remoteAllowed`、`encryptedCodexTasks`、`reasoningEffort`、`serviceTier`）、`optimize`（latency/health/cost/quota 权重）、`limits.maxEstimatedCostUsd`、`unknownEvidence`（allow/penalize/exclude）。未知不会被当作零或免费。 + +CLI：`ocx route policy list`、`ocx route policy show `、`ocx route policy dry-run --model-context --tools`、`ocx route policy evaluate `。 + +组合是显式的有序/加权目标路由与故障转移；策略配置文件是基于证据在候选之间进行选择。 + +## 请求历史与路由分析 + +- `GET /api/request-history` - 从派生索引（`routing-history.sqlite`）进行游标分页的全历史查询。过滤器：`provider`、`model`、`requestedModel`、`status`、`conversationId`、`surface`、`inboundProtocol`、`apiKeyId`、`profileId`、`fallback`、`from`、`to`。 +- `GET /api/request-history/:requestId/route-decision` - 为什么选择此路由（跟踪、候选、排除、分数、配置文件+版本、执行尝试、结果）。 +- `GET /api/routing-analytics` - 成功/失败/取消/回退率、p50/p95/p99 耗时与 TTFT、不完整流率、冷却失败数、每次成功请求的估算成本、覆盖率、置信度、截断标志。 +- `GET /api/routing-profiles`、`POST /api/routing-profiles/dry-run` - 配置文件查看与试运行评估（不发送上游请求）。 + +返回的历史记录与路由决策负载仅暴露已脱敏的请求元数据（例如不透明的 `apiKeyId` 标签）。不包含凭证、原始提示正文或提供商密钥。 + +CLI：`ocx logs explain `、`ocx logs rebuild-index`、`ocx logs index-status`。 + +## 迁移 + +`routingProfiles` 是可选的增量配置：现有配置文件与旧 `usage.jsonl` 行均可原样加载。索引是一次性的——删除后会在下次查询时从 `usage.jsonl` 自动重建。系统不会自动调优。 diff --git a/gui/src/App.tsx b/gui/src/App.tsx index 093809b98..289fa8761 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -7,13 +7,14 @@ import Combos from "./pages/Combos"; import Subagents from "./pages/Subagents"; import Logs from "./pages/Logs"; import Usage from "./pages/Usage"; +import RoutingProfiles from "./pages/RoutingProfiles"; import Storage from "./pages/Storage"; import CodexAuth from "./pages/CodexAuth"; import Integrations from "./pages/Integrations"; import Startup from "./pages/Startup"; import ErrorBoundary from "./components/ErrorBoundary"; import { SidebarGithubRow } from "./components/sidebar-github-row"; -import { IconGrid, IconServer, IconBoxes, IconBot, IconList, IconActivity, IconHardDrive, IconKey, IconMenu, IconSun, IconMoon, IconMonitor, IconGlobe, IconPower, IconTerminal, IconX } from "./icons"; +import { IconGrid, IconServer, IconBoxes, IconBot, IconList, IconActivity, IconHardDrive, IconKey, IconMenu, IconSun, IconMoon, IconMonitor, IconGlobe, IconPower, IconTerminal, IconX, IconRoute } from "./icons"; import { useI18n, useT, LOCALES, type Locale, type TKey } from "./i18n/shared"; import { Select } from "./ui"; import { installApiAuthFetch } from "./api"; @@ -38,6 +39,7 @@ const PAGE_TKEY: Record = { storage: "nav.storage", "codex-auth": "nav.codexAuth", integrations: "nav.integrations", + routing: "nav.routing", }; const API_BASE = import.meta.env.VITE_API_BASE || ""; @@ -66,6 +68,7 @@ const NAV: NavEntry[] = [ { id: "subagents", tkey: "nav.subagents", Icon: IconBot }, { id: "logs", tkey: "nav.logs", Icon: IconList }, { id: "usage", tkey: "nav.usage", Icon: IconActivity }, + { id: "routing", tkey: "nav.routing", Icon: IconRoute }, { id: "storage", tkey: "nav.storage", Icon: IconHardDrive }, /* * Claude sits directly above Integrations because it is a shortcut into that @@ -338,6 +341,7 @@ export default function App() { {page === "subagents" && } {page === "logs" && } {page === "usage" && } + {page === "routing" && } {page === "storage" && } {page === "codex-auth" && } {page === "integrations" && } diff --git a/gui/src/app-routing.ts b/gui/src/app-routing.ts index 82916a014..7fc5cfea2 100644 --- a/gui/src/app-routing.ts +++ b/gui/src/app-routing.ts @@ -13,7 +13,8 @@ export type Page = | "usage" | "storage" | "codex-auth" - | "integrations"; + | "integrations" + | "routing"; export const VALID_PAGES = new Set([ "dashboard", @@ -27,6 +28,7 @@ export const VALID_PAGES = new Set([ "storage", "codex-auth", "integrations", + "routing", ]); export function readPageFromHash(hash?: string): Page { diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 46d78face..1abe5e37e 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -10,7 +10,48 @@ export const de: Record = { "nav.providers": "Anbieter", "nav.models": "Modelle", "nav.combos": "Combos", + "nav.routing": "Routing (beta)", "nav.subagents": "Sub-Agenten", + + // routing intelligence + "routing.title": "Routing-Intelligenz (beta)", + "routing.subtitle": "Policy-Profile, Trockenlauf-Bewertung und routinggestützte Analysen.", + "routing.loadFailed": "Routing-Daten konnten nicht geladen werden", + "routing.empty": "Keine Routing-Profile konfiguriert. Fügen Sie `routingProfiles` zur config.json hinzu.", + "routing.revision": "rev", + "routing.detail": "Profil", + "routing.candidates": "Kandidaten", + "routing.require": "Harte Anforderungen", + "routing.optimize": "Optimierungsgewichte", + "routing.limits": "Grenzen", + "routing.unknownEvidence": "Richtlinie für unbekannte Evidenz", + "routing.none": "keine", + "routing.unavailable": "–", + "routing.dryRun": "Trockenlauf-Bewertung", + "routing.dryRunContext": "Kontextfenster der Anfrage (Tokens)", + "routing.dryRunTools": "Anfrage benötigt Tools", + "routing.dryRunImage": "Anfrage benötigt Bild-Eingabe", + "routing.dryRunStructured": "Anfrage benötigt strukturierte Ausgabe", + "routing.dryRunRun": "Kandidaten bewerten", + "routing.candidate": "Kandidat", + "routing.eligible": "Geeignet", + "routing.exclusions": "Ausschlüsse", + "routing.score": "Punktzahl", + "routing.selected": "ausgewählt", + "routing.yes": "ja", + "routing.no": "nein", + "routing.analytics": "Routing-Analysen", + "routing.analyticsTotal": "Anfragen", + "routing.analyticsSuccessRate": "Erfolg", + "routing.analyticsFallbackRate": "Fallback", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "Cooldown-Fehler", + "routing.analyticsConfidence": "Konfidenz", + "routing.analyticsTruncated": "abgeschnittener Verlauf", + "routing.analyticsRequests": "Anfragen", + "routing.analyticsEmpty": "Noch keine Analysen – senden Sie zuerst einige Anfragen.", "nav.logs": "Protokolle & Diagnose", "nav.usage": "Nutzung", "common.github": "GitHub", @@ -534,6 +575,12 @@ export const de: Record = { "usage.cost.disclaimer": "Kein Abrechnungsbeleg. Stattdessen können Abonnementnutzung oder Anbieter-Guthaben gelten.", "usage.cost.unpricedNote": "{count} Anfragen ohne Preis oder Nutzung ausgeschlossen", "logs.detail.section.basic": "Grundinformationen", + "logs.detail.route.section": "Route-Entscheidung", + "logs.detail.route.kind": "Route-Typ", + "logs.detail.route.profile": "Profil", + "logs.detail.route.selected": "Ausgewählt", + "logs.detail.route.candidates": "Kandidaten", + "logs.detail.route.unknown": "Für diese Anfrage wurde keine Route-Entscheidung aufgezeichnet (Zeile vor dem Trace).", "logs.detail.section.performance": "Leistung", "logs.detail.section.cost": "API-Listenpreis-Äquivalent", "logs.detail.section.attempts": "Combo-Versuche", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 8389a237a..0ad668ca8 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -12,6 +12,7 @@ export const en = { "nav.providers": "Providers", "nav.models": "Models", "nav.combos": "Combos", + "nav.routing": "Routing (beta)", "nav.subagents": "Subagents", "nav.logs": "Logs & Debug", "nav.usage": "Usage", @@ -52,6 +53,46 @@ export const en = { "errorBoundary.details": "Error", "errorBoundary.reload": "Reload", + // routing intelligence + "routing.title": "Routing Intelligence (beta)", + "routing.subtitle": "Policy profiles, dry-run evaluation, and source-backed routing analytics.", + "routing.loadFailed": "Could not load routing data", + "routing.empty": "No routing profiles configured. Add `routingProfiles` to config.json.", + "routing.revision": "rev", + "routing.detail": "Profile", + "routing.candidates": "Candidates", + "routing.require": "Hard requirements", + "routing.optimize": "Optimization weights", + "routing.limits": "Limits", + "routing.unknownEvidence": "Unknown evidence policy", + "routing.none": "none", + "routing.unavailable": "–", + "routing.dryRun": "Dry-run evaluation", + "routing.dryRunContext": "Request context window (tokens)", + "routing.dryRunTools": "Request requires tools", + "routing.dryRunImage": "Request requires image input", + "routing.dryRunStructured": "Request requires structured output", + "routing.dryRunRun": "Evaluate candidates", + "routing.candidate": "Candidate", + "routing.eligible": "Eligible", + "routing.exclusions": "Exclusions", + "routing.score": "Score", + "routing.selected": "selected", + "routing.yes": "yes", + "routing.no": "no", + "routing.analytics": "Routing analytics", + "routing.analyticsTotal": "Requests", + "routing.analyticsSuccessRate": "Success", + "routing.analyticsFallbackRate": "Fallback", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "Cooldown failures", + "routing.analyticsConfidence": "Confidence", + "routing.analyticsTruncated": "truncated history", + "routing.analyticsRequests": "Requests", + "routing.analyticsEmpty": "No analytics yet — send some requests first.", + // startup health "startup.title": "Startup safety", "startup.subtitle": "Verify that Codex can reach opencodex after a restart, before local proxy routing becomes a reconnect loop.", @@ -561,6 +602,12 @@ export const en = { "usage.cost.disclaimer": "Not a billing receipt. Subscription usage or provider credits may apply instead.", "usage.cost.unpricedNote": "{count} requests excluded (no price or usage)", "logs.detail.section.basic": "Basic information", + "logs.detail.route.section": "Route decision", + "logs.detail.route.kind": "Route kind", + "logs.detail.route.profile": "Profile", + "logs.detail.route.selected": "Selected", + "logs.detail.route.candidates": "Candidates", + "logs.detail.route.unknown": "No route trace recorded for this request (pre-trace row).", "logs.detail.section.performance": "Performance", "logs.detail.section.cost": "API list-price equivalent", "logs.detail.section.attempts": "Combo attempts", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 77aaa0dd8..d64005bb5 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -10,7 +10,48 @@ export const ja: Record = { "nav.providers": "プロバイダー", "nav.models": "モデル", "nav.combos": "コンボ", + "nav.routing": "ルーティング (beta)", "nav.subagents": "サブエージェント", + + // routing intelligence + "routing.title": "ルーティングインテリジェンス (beta)", + "routing.subtitle": "ポリシープロファイル、ドライラン評価、ソース連携のルーティング分析。", + "routing.loadFailed": "ルーティングデータを読み込めませんでした", + "routing.empty": "ルーティングプロファイルが設定されていません。config.json に `routingProfiles` を追加してください。", + "routing.revision": "rev", + "routing.detail": "プロファイル", + "routing.candidates": "候補", + "routing.require": "必須要件", + "routing.optimize": "最適化ウェイト", + "routing.limits": "制限", + "routing.unknownEvidence": "不明なエビデンスのポリシー", + "routing.none": "なし", + "routing.unavailable": "–", + "routing.dryRun": "ドライラン評価", + "routing.dryRunContext": "リクエストのコンテキストウィンドウ（トークン）", + "routing.dryRunTools": "リクエストにツールが必要", + "routing.dryRunImage": "リクエストに画像入力が必要", + "routing.dryRunStructured": "リクエストに構造化出力が必要", + "routing.dryRunRun": "候補を評価", + "routing.candidate": "候補", + "routing.eligible": "対象", + "routing.exclusions": "除外", + "routing.score": "スコア", + "routing.selected": "選択済み", + "routing.yes": "はい", + "routing.no": "いいえ", + "routing.analytics": "ルーティング分析", + "routing.analyticsTotal": "リクエスト", + "routing.analyticsSuccessRate": "成功率", + "routing.analyticsFallbackRate": "フォールバック", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "クールダウン失敗", + "routing.analyticsConfidence": "信頼度", + "routing.analyticsTruncated": "切り捨て履歴", + "routing.analyticsRequests": "リクエスト", + "routing.analyticsEmpty": "分析はまだありません。まずリクエストを送信してください。", "nav.logs": "ログ & デバッグ", "nav.usage": "使用量", "common.github": "GitHub", @@ -519,6 +560,12 @@ export const ja: Record = { "usage.cost.disclaimer": "請求明細ではありません。サブスクリプション利用量やプロバイダークレジットが代わりに適用される場合があります。", "usage.cost.unpricedNote": "{count} 件のリクエストを除外(価格または使用量なし)", "logs.detail.section.basic": "基本情報", + "logs.detail.route.section": "ルート決定", + "logs.detail.route.kind": "ルート種別", + "logs.detail.route.profile": "プロファイル", + "logs.detail.route.selected": "選択済み", + "logs.detail.route.candidates": "候補", + "logs.detail.route.unknown": "このリクエストにはルートトレースが記録されていません（トレース前の行）。", "logs.detail.section.performance": "パフォーマンス", "logs.detail.section.cost": "API 定価相当額", "logs.detail.section.attempts": "コンボの試行", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 93e03755e..2205702c6 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -10,7 +10,48 @@ export const ko: Record = { "nav.providers": "프로바이더", "nav.models": "모델", "nav.combos": "콤보", + "nav.routing": "라우팅 (beta)", "nav.subagents": "서브에이전트", + + // routing intelligence + "routing.title": "라우팅 인텔리전스 (beta)", + "routing.subtitle": "정책 프로필, 드라이런 평가, 소스 기반 라우팅 분석.", + "routing.loadFailed": "라우팅 데이터를 불러오지 못했습니다", + "routing.empty": "라우팅 프로필이 구성되지 않았습니다. config.json에 `routingProfiles`를 추가하세요.", + "routing.revision": "rev", + "routing.detail": "프로필", + "routing.candidates": "후보", + "routing.require": "필수 요구사항", + "routing.optimize": "최적화 가중치", + "routing.limits": "제한", + "routing.unknownEvidence": "알 수 없는 증거 정책", + "routing.none": "없음", + "routing.unavailable": "–", + "routing.dryRun": "드라이런 평가", + "routing.dryRunContext": "요청 컨텍스트 창(토큰)", + "routing.dryRunTools": "요청에 도구 필요", + "routing.dryRunImage": "요청에 이미지 입력 필요", + "routing.dryRunStructured": "요청에 구조화된 출력 필요", + "routing.dryRunRun": "후보 평가", + "routing.candidate": "후보", + "routing.eligible": "적격", + "routing.exclusions": "제외", + "routing.score": "점수", + "routing.selected": "선택됨", + "routing.yes": "예", + "routing.no": "아니요", + "routing.analytics": "라우팅 분석", + "routing.analyticsTotal": "요청", + "routing.analyticsSuccessRate": "성공", + "routing.analyticsFallbackRate": "폴백", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "쿨다운 실패", + "routing.analyticsConfidence": "신뢰도", + "routing.analyticsTruncated": "잘린 기록", + "routing.analyticsRequests": "요청", + "routing.analyticsEmpty": "분석이 아직 없습니다. 먼저 요청을 보내세요.", "nav.logs": "로그&디버그", "nav.usage": "사용량", "common.github": "GitHub", @@ -553,6 +594,12 @@ export const ko: Record = { "usage.cost.disclaimer": "결제 영수증이 아닙니다. 구독 사용량 또는 프로바이더 크레딧이 대신 적용될 수 있습니다.", "usage.cost.unpricedNote": "비용 산정 불가 {count}건 제외", "logs.detail.section.basic": "기본 정보", + "logs.detail.route.section": "라우팅 결정", + "logs.detail.route.kind": "라우팅 종류", + "logs.detail.route.profile": "프로필", + "logs.detail.route.selected": "선택됨", + "logs.detail.route.candidates": "후보", + "logs.detail.route.unknown": "이 요청에 대한 라우팅 추적이 기록되지 않았습니다(추적 이전 행).", "logs.detail.section.performance": "성능", "logs.detail.section.cost": "API 정가 환산치", "logs.detail.section.attempts": "Combo 시도", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 48e9149b6..47dd556ea 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -10,7 +10,48 @@ export const ru: Record = { "nav.providers": "Провайдеры", "nav.models": "Модели", "nav.combos": "Комбо", + "nav.routing": "Маршрутизация (beta)", "nav.subagents": "Подагенты", + + // routing intelligence + "routing.title": "Интеллект маршрутизации (beta)", + "routing.subtitle": "Политики маршрутизации, пробная оценка и аналитика на основе источников.", + "routing.loadFailed": "Не удалось загрузить данные маршрутизации", + "routing.empty": "Профили маршрутизации не настроены. Добавьте `routingProfiles` в config.json.", + "routing.revision": "rev", + "routing.detail": "Профиль", + "routing.candidates": "Кандидаты", + "routing.require": "Жёсткие требования", + "routing.optimize": "Веса оптимизации", + "routing.limits": "Лимиты", + "routing.unknownEvidence": "Политика неизвестных данных", + "routing.none": "нет", + "routing.unavailable": "–", + "routing.dryRun": "Пробная оценка", + "routing.dryRunContext": "Контекстное окно запроса (токены)", + "routing.dryRunTools": "Запрос требует инструменты", + "routing.dryRunImage": "Запрос требует изображения", + "routing.dryRunStructured": "Запрос требует структурированный вывод", + "routing.dryRunRun": "Оценить кандидатов", + "routing.candidate": "Кандидат", + "routing.eligible": "Допустим", + "routing.exclusions": "Исключения", + "routing.score": "Оценка", + "routing.selected": "выбран", + "routing.yes": "да", + "routing.no": "нет", + "routing.analytics": "Аналитика маршрутизации", + "routing.analyticsTotal": "Запросы", + "routing.analyticsSuccessRate": "Успех", + "routing.analyticsFallbackRate": "Фолбэк", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "Сбои кулдауна", + "routing.analyticsConfidence": "Доверие", + "routing.analyticsTruncated": "усечённая история", + "routing.analyticsRequests": "Запросы", + "routing.analyticsEmpty": "Аналитики пока нет — сначала отправьте несколько запросов.", "nav.logs": "Логи и отладка", "nav.usage": "Использование", "common.github": "GitHub", @@ -551,6 +592,12 @@ export const ru: Record = { "usage.cost.disclaimer": "Не является счётом. Расходы могут покрываться подпиской или кредитами провайдера.", "usage.cost.unpricedNote": "Исключено {count} запросов (нет цены или данных использования)", "logs.detail.section.basic": "Основная информация", + "logs.detail.route.section": "Решение о маршруте", + "logs.detail.route.kind": "Тип маршрута", + "logs.detail.route.profile": "Профиль", + "logs.detail.route.selected": "Выбрано", + "logs.detail.route.candidates": "Кандидаты", + "logs.detail.route.unknown": "Для этого запроса трасса маршрута не записана (строка до трассировки).", "logs.detail.section.performance": "Производительность", "logs.detail.section.cost": "Эквивалент стоимости по прайс-листу API", "logs.detail.section.attempts": "Попытки комбо", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 15e6053b3..e279ddf43 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -10,7 +10,48 @@ export const zh: Record = { "nav.providers": "提供方", "nav.models": "模型", "nav.combos": "组合", + "nav.routing": "路由 (beta)", "nav.subagents": "子代理", + + // routing intelligence + "routing.title": "路由智能 (beta)", + "routing.subtitle": "策略配置文件、试运行评估以及基于来源的路由分析。", + "routing.loadFailed": "无法加载路由数据", + "routing.empty": "未配置路由策略。请在 config.json 中添加 `routingProfiles`。", + "routing.revision": "rev", + "routing.detail": "配置文件", + "routing.candidates": "候选", + "routing.require": "硬性要求", + "routing.optimize": "优化权重", + "routing.limits": "限制", + "routing.unknownEvidence": "未知证据策略", + "routing.none": "无", + "routing.unavailable": "–", + "routing.dryRun": "试运行评估", + "routing.dryRunContext": "请求上下文窗口（令牌）", + "routing.dryRunTools": "请求需要工具", + "routing.dryRunImage": "请求需要图像输入", + "routing.dryRunStructured": "请求需要结构化输出", + "routing.dryRunRun": "评估候选", + "routing.candidate": "候选", + "routing.eligible": "合格", + "routing.exclusions": "排除项", + "routing.score": "分数", + "routing.selected": "已选择", + "routing.yes": "是", + "routing.no": "否", + "routing.analytics": "路由分析", + "routing.analyticsTotal": "请求", + "routing.analyticsSuccessRate": "成功率", + "routing.analyticsFallbackRate": "回退", + "routing.analyticsP50": "p50", + "routing.analyticsP95": "p95", + "routing.analyticsP99": "p99", + "routing.analyticsCooldown": "冷却失败", + "routing.analyticsConfidence": "置信度", + "routing.analyticsTruncated": "历史已截断", + "routing.analyticsRequests": "请求", + "routing.analyticsEmpty": "暂无分析 — 请先发送一些请求。", "nav.logs": "日志与调试", "nav.usage": "用量", "common.github": "GitHub", @@ -546,6 +587,12 @@ export const zh: Record = { "usage.cost.disclaimer": "这不是账单或扣费凭证。实际可能计入订阅用量或消耗服务商额度。", "usage.cost.unpricedNote": "已排除 {count} 个无法计费的请求", "logs.detail.section.basic": "基本信息", + "logs.detail.route.section": "路由决策", + "logs.detail.route.kind": "路由类型", + "logs.detail.route.profile": "配置文件", + "logs.detail.route.selected": "已选择", + "logs.detail.route.candidates": "候选", + "logs.detail.route.unknown": "此请求未记录路由跟踪（跟踪之前的行）。", "logs.detail.section.performance": "性能", "logs.detail.section.cost": "API 标价折算", "logs.detail.section.attempts": "Combo 尝试", diff --git a/gui/src/icons.tsx b/gui/src/icons.tsx index bed849259..2705f679f 100644 --- a/gui/src/icons.tsx +++ b/gui/src/icons.tsx @@ -38,6 +38,7 @@ export const IconKey = (p: P) => (); export const IconTicket = (p: P) => (); +export const IconRoute = (p: P) => (); export const IconLink = (p: P) => (); export const IconSun = (p: P) => (); export const IconMoon = (p: P) => (); diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index ebe177532..b513d754f 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -17,6 +17,10 @@ import { logsTabKeyDown, readTabFromHash, selectLogsTab } from "./logs-tab-keydo import { speedLabel } from "./logs-speed-label"; import type { LogSurface, LogSurfaceFilter } from "./logs-surface-filter"; import { logMatchesSurface } from "./logs-surface-filter"; +import { + sanitizeLogEntryRouteDecision, + validCachedRouteDecision, +} from "./log-route-decision"; function logsCacheKey(apiBase: string): string { return `ocx.logs.list.v1:${apiBase}`; @@ -139,9 +143,15 @@ export interface LogEntry { firstOutputMs?: number; attempts?: LogAttempt[]; displayMetrics?: LogDisplayMetrics; + /** Bounded route-decision trace (RI-01); absent for pre-trace rows. */ + routeDecision?: { + routeKind?: string; + profile?: { id?: string; revision?: string }; + selected?: { provider?: string; model?: string; reason?: string }; + candidates?: Array<{ provider?: string; model?: string; eligible?: boolean; exclusions?: Array<{ code?: string }> }>; + }; } -/** Session-cache entries are arbitrary JSON — reject shapes that would crash the table. */ function validCachedLogs(cached: LogEntry[] | null): LogEntry[] | null { if (!Array.isArray(cached)) return null; for (const entry of cached) { @@ -153,6 +163,7 @@ function validCachedLogs(cached: LogEntry[] | null): LogEntry[] | null { || typeof entry.provider !== "string" || typeof entry.status !== "number" || typeof entry.durationMs !== "number" + || !validCachedRouteDecision(entry.routeDecision) ) { return null; } @@ -457,7 +468,8 @@ export default function Logs({ apiBase }: { apiBase: string }) { const res = await fetch(`${apiBase}/api/logs?limit=2000`, { signal }); if (!res.ok) throw new Error(`${res.status} ${res.statusText}`.trim()); const body = await res.json() as LogEntry[] | { logs?: LogEntry[] }; - const next = Array.isArray(body) ? body : (body.logs ?? []); + const raw = Array.isArray(body) ? body : (body.logs ?? []); + const next = raw.map(sanitizeLogEntryRouteDecision); writeSessionListCache(resourceKey, next); return next; }, [apiBase, resourceKey]); @@ -921,6 +933,45 @@ function LogDetailDialog({ +
+

{t("logs.detail.route.section")}

+ {detail.routeDecision ? ( +
+ {t("logs.detail.route.kind")}{detail.routeDecision.routeKind ?? "–"} + {detail.routeDecision.profile?.id && ( + <>{t("logs.detail.route.profile")} + {detail.routeDecision.profile.id} ({detail.routeDecision.profile.revision}) + )} + {detail.routeDecision.selected?.provider && ( + <>{t("logs.detail.route.selected")} + + {detail.routeDecision.selected.provider}/{detail.routeDecision.selected.model} + {detail.routeDecision.selected.reason ? ` — ${detail.routeDecision.selected.reason}` : ""} + + )} + {t("logs.detail.route.candidates")} + + {(detail.routeDecision.candidates ?? []).map(candidate => { + const provider = typeof candidate.provider === "string" && candidate.provider.length > 0 + ? candidate.provider + : "–"; + const model = typeof candidate.model === "string" && candidate.model.length > 0 + ? candidate.model + : "–"; + const mark = candidate.eligible === true + ? " ✓" + : candidate.eligible === false + ? " ✗" + : " ?"; + return `${provider}/${model}${mark}`; + }).join(" ") || "–"} + +
+ ) : ( +

{t("logs.detail.route.unknown")}

+ )} +
+

{t("logs.detail.section.performance")}

diff --git a/gui/src/pages/RoutingProfiles.tsx b/gui/src/pages/RoutingProfiles.tsx new file mode 100644 index 000000000..7732dabf0 --- /dev/null +++ b/gui/src/pages/RoutingProfiles.tsx @@ -0,0 +1,373 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Notice } from "../ui"; +import { useT } from "../i18n/shared"; + +type ProfileDto = { + id: string; + model: string; + revision: string; + candidates: Array<{ provider: string; model: string }>; + require: Record; + optimize: Record; + limits: Record; + unknownEvidence: Record; +}; + +type DryRunCandidate = { + provider: string; + model: string; + eligible: boolean; + exclusions: Array<{ code: string; detail?: string }>; + score?: { total: number; components: Record }; +}; + +type Analytics = { + totalRequests: number; + successRate: number | null; + fallbackRate: number | null; + confidence: string | null; + historyTruncated: boolean; + cooldownTriggeringFailures: number; + durationMs: { p50?: number; p95?: number; p99?: number; sampleCount: number }; + firstOutputMs: { p50?: number; p95?: number; p99?: number; sampleCount: number; coverage: number | null }; + breakdown: Array<{ provider: string; model: string; requests: number; successRate: number | null; p50DurationMs?: number }>; +}; + +type DryRunResult = { + candidates: DryRunCandidate[]; + selectedIndex: number | null; + trace?: { profile?: { revision?: string } }; +}; + +function fmtMs(value: number | undefined, unavailable: string): string { + return value === undefined ? unavailable : `${Math.round(value)}ms`; +} + +function fmtRate(value: number | null | undefined, unavailable: string): string { + return value === null || value === undefined ? unavailable : `${Math.round(value * 100)}%`; +} + +function pickSelectedProfile(next: ProfileDto[], current: ProfileDto | null): ProfileDto | null { + if (current) { + const refreshed = next.find(profile => profile.id === current.id); + if (refreshed) return refreshed; + } + return next[0] ?? null; +} + +function shouldClearDryRunOnSelectionChange( + current: ProfileDto | null, + next: ProfileDto | null, +): boolean { + if (!current) return false; + if (!next) return true; + return current.id !== next.id || current.revision !== next.revision; +} + +export default function RoutingProfiles({ apiBase }: { apiBase: string }) { + const t = useT(); + const unavailable = t("routing.unavailable"); + const [profiles, setProfiles] = useState([]); + const [analytics, setAnalytics] = useState(null); + const [loadError, setLoadError] = useState(""); + const [selected, setSelected] = useState(null); + const [context, setContext] = useState(""); + const [tools, setTools] = useState(false); + const [image, setImage] = useState(false); + const [structured, setStructured] = useState(false); + const [dryRunResult, setDryRunResult] = useState(null); + const [dryRunError, setDryRunError] = useState(""); + const [running, setRunning] = useState(false); + const selectedRef = useRef(null); + const dryRunGenerationRef = useRef(0); + + const clearDryRun = useCallback(() => { + dryRunGenerationRef.current += 1; + setDryRunResult(null); + setDryRunError(""); + }, []); + + const selectProfile = useCallback((profile: ProfileDto | null) => { + selectedRef.current = profile; + setSelected(profile); + clearDryRun(); + }, [clearDryRun]); + + const loadGenerationRef = useRef(0); + + const load = useCallback(async () => { + const generation = ++loadGenerationRef.current; + setLoadError(""); + try { + const [profilesRes, analyticsRes] = await Promise.all([ + fetch(`${apiBase}/api/routing-profiles`), + fetch(`${apiBase}/api/routing-analytics`), + ]); + if (generation !== loadGenerationRef.current) return; + if (!profilesRes.ok) throw new Error(`load-${profilesRes.status}`); + const profilesJson = await profilesRes.json() as { profiles?: ProfileDto[] }; + if (generation !== loadGenerationRef.current) return; + let analyticsJson: Analytics | null = null; + if (analyticsRes.ok) { + analyticsJson = await analyticsRes.json() as Analytics; + if (generation !== loadGenerationRef.current) return; + } + // Apply state only after every body await, and only while this load is still current. + if (generation !== loadGenerationRef.current) return; + const next = profilesJson.profiles ?? []; + const current = selectedRef.current; + const refreshed = pickSelectedProfile(next, current); + selectedRef.current = refreshed; + setProfiles(next); + setSelected(refreshed); + if (shouldClearDryRunOnSelectionChange(current, refreshed)) { + clearDryRun(); + } + setAnalytics(analyticsJson); + } catch (error) { + if (generation !== loadGenerationRef.current) return; + setLoadError(error instanceof Error ? error.message : String(error)); + } + }, [apiBase, clearDryRun]); + + useEffect(() => { + const timer = window.setTimeout(() => void load(), 0); + return () => window.clearTimeout(timer); + }, [load]); + + const runDryRun = async () => { + if (!selected) return; + const generation = ++dryRunGenerationRef.current; + setRunning(true); + setDryRunResult(null); + setDryRunError(""); + try { + const evidence: Record = {}; + const contextTokens = context.trim() ? Number(context.trim()) : NaN; + if (Number.isFinite(contextTokens) && contextTokens > 0) { + evidence.contextWindow = contextTokens; + } + if (tools) evidence.toolsRequired = true; + if (image) evidence.imageInputRequired = true; + if (structured) evidence.structuredOutputRequired = true; + const response = await fetch(`${apiBase}/api/routing-profiles/dry-run`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: selected.id, + evidence, + }), + }); + if (generation !== dryRunGenerationRef.current) return; + if (!response.ok) { + let message = `dry-run ${response.status}`; + try { + const body = await response.json() as { error?: { message?: string } }; + message = body.error?.message ?? message; + } catch { + // Keep the status fallback when the error body is not JSON. + } + if (generation !== dryRunGenerationRef.current) return; + setDryRunError(message); + return; + } + const result = await response.json() as DryRunResult; + if (generation !== dryRunGenerationRef.current) return; + setDryRunResult(result); + } catch (error) { + if (generation !== dryRunGenerationRef.current) return; + setDryRunError(error instanceof Error ? error.message : String(error)); + } finally { + if (generation === dryRunGenerationRef.current) { + setRunning(false); + } + } + }; + + return ( +
+
+

{t("routing.title")}

+ +
+

{t("routing.subtitle")}

+ + {loadError ? {t("routing.loadFailed")}: {loadError} : null} + + {profiles.length === 0 && !loadError ? ( +
{t("routing.empty")}
+ ) : ( +
+ {profiles.map(profile => ( + + ))} +
+ )} + + {selected ? ( +
+

{t("routing.detail")}: {selected.model}

+
+ {t("routing.candidates")} +
+ {selected.candidates.map(candidate => ( +
+ {candidate.provider}/{candidate.model} +
+ ))} +
+
+ {([ + ["routing.require", selected.require, true], + ["routing.optimize", selected.optimize, false], + ["routing.limits", selected.limits, true], + ["routing.unknownEvidence", selected.unknownEvidence, false], + ] as const).map(([labelKey, value, allowEmpty]) => ( +
+ {t(labelKey)} + 
+                {allowEmpty && Object.keys(value).length === 0
+                  ? t("routing.none")
+                  : JSON.stringify(value, null, 2)}
+
+
+ ))} +
+ ) : null} + +
+

{t("routing.dryRun")}

+ + + + + + {dryRunError ? {dryRunError} : null} + {dryRunResult ? ( + + + + + + + + + + + {dryRunResult.candidates.map((candidate, index) => ( + + + + + + + ))} + +
{t("routing.candidate")}{t("routing.eligible")}{t("routing.exclusions")}{t("routing.score")}
+ {candidate.provider}/{candidate.model} + {index === dryRunResult.selectedIndex ? ` ✓ (${t("routing.selected")})` : ""} + {candidate.eligible ? t("routing.yes") : t("routing.no")}{candidate.exclusions.map(exclusion => exclusion.code).join(", ") || t("routing.none")}{candidate.score ? candidate.score.total.toFixed(3) : unavailable}
+ ) : null} +
+ +
+

{t("routing.analytics")}

+ {analytics ? ( + <> +
+ {t("routing.analyticsTotal")}: {analytics.totalRequests} + {t("routing.analyticsSuccessRate")}: {fmtRate(analytics.successRate, unavailable)} + {t("routing.analyticsFallbackRate")}: {fmtRate(analytics.fallbackRate, unavailable)} + {t("routing.analyticsP50")}: {fmtMs(analytics.durationMs.p50, unavailable)} + {t("routing.analyticsP95")}: {fmtMs(analytics.durationMs.p95, unavailable)} + {t("routing.analyticsP99")}: {fmtMs(analytics.durationMs.p99, unavailable)} + {t("routing.analyticsCooldown")}: {analytics.cooldownTriggeringFailures} + {t("routing.analyticsConfidence")}: {analytics.confidence ?? unavailable} + {analytics.historyTruncated ? {t("routing.analyticsTruncated")} : null} +
+ + + + + + + + + + + {analytics.breakdown.map(row => ( + + + + + + + ))} + +
{t("routing.candidate")}{t("routing.analyticsRequests")}{t("routing.analyticsSuccessRate")}{t("routing.analyticsP50")}
{row.provider}/{row.model}{row.requests}{fmtRate(row.successRate, unavailable)}{fmtMs(row.p50DurationMs, unavailable)}
+ + ) : ( +

{t("routing.analyticsEmpty")}

+ )} +
+
+ ); +} diff --git a/gui/src/pages/log-route-decision.ts b/gui/src/pages/log-route-decision.ts new file mode 100644 index 000000000..dbf606014 --- /dev/null +++ b/gui/src/pages/log-route-decision.ts @@ -0,0 +1,64 @@ +/** Shared route-decision validation for Logs session-cache / API rows. */ + +export interface LogRouteDecision { + routeKind?: string; + profile?: { id?: string; revision?: string }; + selected?: { provider?: string; model?: string; reason?: string }; + candidates?: Array<{ provider?: string; model?: string; eligible?: boolean; exclusions?: Array<{ code?: string }> }>; +} + +export interface LogEntryBase { + requestId?: string; + timestamp: number; + model: string; + provider: string; + status: number; + durationMs: number; + routeDecision?: LogRouteDecision; +} + +function isOptionalString(value: unknown): boolean { + return value === undefined || typeof value === "string"; +} + +/** Session-cache / API entries are arbitrary JSON — reject shapes that would crash the detail panel. */ +export function validCachedRouteDecision(routeDecision: LogRouteDecision | undefined): boolean { + if (routeDecision === undefined) return true; + if (!routeDecision || typeof routeDecision !== "object") return false; + if (!isOptionalString(routeDecision.routeKind)) return false; + if (routeDecision.profile !== undefined) { + if (!routeDecision.profile || typeof routeDecision.profile !== "object") return false; + if (!isOptionalString(routeDecision.profile.id)) return false; + if (!isOptionalString(routeDecision.profile.revision)) return false; + } + if (routeDecision.selected !== undefined) { + if (!routeDecision.selected || typeof routeDecision.selected !== "object") return false; + if (!isOptionalString(routeDecision.selected.provider)) return false; + if (!isOptionalString(routeDecision.selected.model)) return false; + if (!isOptionalString(routeDecision.selected.reason)) return false; + } + if (routeDecision.candidates === undefined) return true; + if (!Array.isArray(routeDecision.candidates)) return false; + for (const candidate of routeDecision.candidates) { + if (!candidate || typeof candidate !== "object") return false; + if (!isOptionalString(candidate.provider)) return false; + if (!isOptionalString(candidate.model)) return false; + if (candidate.eligible !== undefined && typeof candidate.eligible !== "boolean") return false; + if (candidate.exclusions !== undefined) { + if (!Array.isArray(candidate.exclusions)) return false; + for (const exclusion of candidate.exclusions) { + if (!exclusion || typeof exclusion !== "object") return false; + if (!isOptionalString(exclusion.code)) return false; + } + } + } + return true; +} + +export function sanitizeLogEntryRouteDecision(entry: T): T { + if (entry.routeDecision === undefined) return entry; + if (validCachedRouteDecision(entry.routeDecision)) return entry; + const rest = { ...entry }; + delete rest.routeDecision; + return rest; +} diff --git a/gui/src/styles.css b/gui/src/styles.css index c133bef9e..6e2be3dca 100644 --- a/gui/src/styles.css +++ b/gui/src/styles.css @@ -960,6 +960,8 @@ a.btn, a.btn:hover { text-decoration: none; } .tbl tbody td { padding: 10px 12px; border-bottom: 1px solid var(--border-soft); } .tbl tbody tr:last-child td { border-bottom: none; } .tbl tbody tr:hover td { background: var(--hover); } + +.checkbox { display: flex; align-items: center; gap: 8px; cursor: pointer; } .tbl .num { text-align: right; font-family: var(--font-code); font-variant-numeric: tabular-nums; } .tbl-wrap { border: 1px solid var(--border); diff --git a/gui/tests/routing-profiles.test.tsx b/gui/tests/routing-profiles.test.tsx new file mode 100644 index 000000000..64bc1863b --- /dev/null +++ b/gui/tests/routing-profiles.test.tsx @@ -0,0 +1,347 @@ +/** @jsxImportSource react */ +// CI path-filter retrigger for RI-10 bot gates. +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { Window } from "happy-dom"; +import { act } from "react"; +import type { Root } from "react-dom/client"; +import { LanguageProvider } from "../src/i18n/provider"; +import RoutingProfiles from "../src/pages/RoutingProfiles"; + +const originalFetch = globalThis.fetch; +let restoreGlobals: (() => void) | undefined; +let previousLanguageDescriptor: PropertyDescriptor | undefined; +let testWindow: Window; + +const PROFILE = { + id: "balanced", + model: "policy/balanced", + revision: "rev-abc", + candidates: [ + { provider: "openai", model: "gpt-5" }, + { provider: "anthropic", model: "claude-sonnet-4" }, + ], + require: { tools: true }, + optimize: { latency: 0.4, health: 0.6 }, + limits: { maxEstimatedCostUsd: 0.5 }, + unknownEvidence: { capability: "exclude", health: "penalize" }, +}; + +const PROFILE_B = { + ...PROFILE, + id: "cheap", + model: "policy/cheap", + revision: "rev-cheap", + candidates: [{ provider: "openai", model: "gpt-5-mini" }], +}; + +const PROFILE_REFRESHED = { + ...PROFILE, + revision: "rev-def", + require: { tools: true, imageInput: true }, +}; + +const ANALYTICS = { + totalRequests: 12, + successRate: 0.75, + fallbackRate: 0.1, + confidence: "medium", + historyTruncated: true, + cooldownTriggeringFailures: 2, + durationMs: { p50: 120, p95: 400, p99: 900, sampleCount: 12 }, + firstOutputMs: { p50: 40, p95: 90, p99: 180, sampleCount: 10, coverage: 0.8 }, + breakdown: [ + { provider: "openai", model: "gpt-5", requests: 8, successRate: 0.875, p50DurationMs: 100 }, + { provider: "anthropic", model: "claude-sonnet-4", requests: 4, successRate: 0.5, p50DurationMs: 180 }, + ], +}; + +const DRY_RUN_OK = { + selectedIndex: 0, + candidates: [ + { + provider: "openai", + model: "gpt-5", + eligible: true, + exclusions: [], + score: { total: 0.91, components: { health: 0.5, latency: 0.41 } }, + }, + { + provider: "anthropic", + model: "claude-sonnet-4", + eligible: false, + exclusions: [{ code: "unknown-capability", detail: "tools" }], + score: { total: 0.2, components: { health: 0.2 } }, + }, + ], + trace: { profile: { revision: "rev-abc" } }, +}; + +beforeEach(() => { + testWindow = new Window({ url: "http://localhost/" }); + previousLanguageDescriptor = Object.getOwnPropertyDescriptor(globalThis.navigator, "language"); + Object.defineProperty(globalThis.navigator, "language", { configurable: true, value: "en-US" }); + const keys = ["document", "window", "localStorage", "IS_REACT_ACT_ENVIRONMENT"] as const; + const previous = Object.fromEntries( + keys.map(key => [key, Object.getOwnPropertyDescriptor(globalThis, key)]), + ) as Record<(typeof keys)[number], PropertyDescriptor | undefined>; + Object.defineProperties(globalThis, { + document: { configurable: true, value: testWindow.document }, + window: { configurable: true, value: testWindow }, + localStorage: { configurable: true, value: testWindow.localStorage }, + IS_REACT_ACT_ENVIRONMENT: { configurable: true, value: true }, + }); + restoreGlobals = () => { + for (const key of keys) { + const descriptor = previous[key]; + if (descriptor) Object.defineProperty(globalThis, key, descriptor); + else delete (globalThis as Record)[key]; + } + if (previousLanguageDescriptor) { + Object.defineProperty(globalThis.navigator, "language", previousLanguageDescriptor); + } else { + delete (globalThis.navigator as { language?: string }).language; + } + }; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + restoreGlobals?.(); + testWindow.close(); +}); + +async function tick(times = 2): Promise { + for (let i = 0; i < times; i++) { + await act(async () => { + await new Promise(resolve => testWindow.setTimeout(resolve, 0)); + await Promise.resolve(); + }); + } +} + +function installFetch(handler: (url: string, init?: RequestInit) => Response | Promise): void { + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + return handler(String(input), init); + }) as typeof fetch; +} + +async function mountPage(): Promise<{ container: HTMLDivElement; root: Root }> { + const container = testWindow.document.createElement("div") as unknown as HTMLDivElement; + testWindow.document.body.appendChild(container); + const { createRoot } = await import("react-dom/client"); + let root!: Root; + await act(async () => { + root = createRoot(container); + root.render( + + + , + ); + }); + await tick(3); + return { container, root }; +} + +test("routing page loads profiles, analytics, and marks the dry-run selection", async () => { + const dryRunBodies: unknown[] = []; + installFetch((url, init) => { + if (url.endsWith("/api/routing-profiles") && (init?.method ?? "GET") === "GET") { + return Response.json({ profiles: [PROFILE] }); + } + if (url.endsWith("/api/routing-analytics")) { + return Response.json(ANALYTICS); + } + if (url.endsWith("/api/routing-profiles/dry-run") && init?.method === "POST") { + dryRunBodies.push(JSON.parse(String(init.body ?? "{}"))); + return Response.json(DRY_RUN_OK); + } + return new Response("missing", { status: 404 }); + }); + + const { container, root } = await mountPage(); + try { + expect(container.querySelector('[data-page="routing"]')).toBeTruthy(); + expect(container.textContent).toContain("Routing Intelligence (beta)"); + expect(container.textContent).toContain("balanced"); + expect(container.textContent).toContain("policy/balanced"); + expect(container.textContent).toContain("rev-abc"); + expect(container.textContent).toContain("openai/gpt-5"); + expect(container.textContent).toContain("Requests: 12"); + expect(container.textContent).toContain("Success: 75%"); + expect(container.textContent).toContain("truncated history"); + + const evaluate = [...container.querySelectorAll("button")] + .find(button => button.textContent?.includes("Evaluate candidates")); + expect(evaluate).toBeTruthy(); + await act(async () => { evaluate!.click(); }); + await tick(3); + + expect(dryRunBodies).toEqual([{ profile: "balanced", evidence: {} }]); + expect(container.textContent).toContain("selected"); + expect(container.textContent).toContain("unknown-capability"); + expect(container.textContent).toContain("0.910"); + } finally { + await act(async () => { root.unmount(); }); + } +}); + +test("routing dry-run checks response status before treating the body as success", async () => { + installFetch((url, init) => { + if (url.endsWith("/api/routing-profiles") && (init?.method ?? "GET") === "GET") { + return Response.json({ profiles: [PROFILE] }); + } + if (url.endsWith("/api/routing-analytics")) { + return Response.json(ANALYTICS); + } + if (url.endsWith("/api/routing-profiles/dry-run") && init?.method === "POST") { + return Response.json({ error: { message: "profile missing" } }, { status: 404 }); + } + return new Response("missing", { status: 404 }); + }); + + const { container, root } = await mountPage(); + try { + const evaluate = [...container.querySelectorAll("button")] + .find(button => button.textContent?.includes("Evaluate candidates")); + expect(evaluate).toBeTruthy(); + await act(async () => { evaluate!.click(); }); + await tick(3); + expect(container.textContent).toContain("profile missing"); + expect(container.textContent).not.toContain("0.910"); + } finally { + await act(async () => { root.unmount(); }); + } +}); + +test("routing clears stale dry-run results when switching profiles", async () => { + installFetch((url, init) => { + if (url.endsWith("/api/routing-profiles") && (init?.method ?? "GET") === "GET") { + return Response.json({ profiles: [PROFILE, PROFILE_B] }); + } + if (url.endsWith("/api/routing-analytics")) { + return Response.json(ANALYTICS); + } + if (url.endsWith("/api/routing-profiles/dry-run") && init?.method === "POST") { + return Response.json(DRY_RUN_OK); + } + return new Response("missing", { status: 404 }); + }); + + const { container, root } = await mountPage(); + try { + const evaluate = [...container.querySelectorAll("button")] + .find(button => button.textContent?.includes("Evaluate candidates")); + expect(evaluate).toBeTruthy(); + await act(async () => { evaluate!.click(); }); + await tick(3); + expect(container.textContent).toContain("0.910"); + + const cheap = [...container.querySelectorAll("button")] + .find(button => button.textContent?.includes("cheap")); + expect(cheap).toBeTruthy(); + await act(async () => { cheap!.click(); }); + await tick(2); + + expect(container.textContent).toContain("policy/cheap"); + expect(container.textContent).not.toContain("0.910"); + expect(container.textContent).not.toContain("unknown-capability"); + } finally { + await act(async () => { root.unmount(); }); + } +}); + +test("routing refreshes the selected profile after reload", async () => { + let profilesPayload: unknown[] = [PROFILE]; + installFetch((url, init) => { + if (url.endsWith("/api/routing-profiles") && (init?.method ?? "GET") === "GET") { + return Response.json({ profiles: profilesPayload }); + } + if (url.endsWith("/api/routing-analytics")) { + return Response.json(ANALYTICS); + } + return new Response("missing", { status: 404 }); + }); + + const { container, root } = await mountPage(); + try { + expect(container.textContent).toContain("rev-abc"); + profilesPayload = [PROFILE_REFRESHED]; + const retry = [...container.querySelectorAll("button")] + .find(button => button.textContent?.trim() === "Retry"); + expect(retry).toBeTruthy(); + await act(async () => { retry!.click(); }); + await tick(3); + expect(container.textContent).toContain("rev-def"); + expect(container.textContent).toContain("\"imageInput\": true"); + } finally { + await act(async () => { root.unmount(); }); + } +}); + + +test("routing ignores a stale load body that finishes after a newer retry", async () => { + let profilesWave = 0; + let releaseStale!: () => void; + const staleGate = new Promise(resolve => { + releaseStale = resolve; + }); + + installFetch((url, init) => { + if (url.endsWith("/api/routing-profiles") && (init?.method ?? "GET") === "GET") { + profilesWave += 1; + const wave = profilesWave; + if (wave === 1) return Response.json({ profiles: [PROFILE] }); + if (wave === 2) { + return { + ok: true, + status: 200, + async json() { + await staleGate; + return { profiles: [PROFILE_B] }; + }, + } as unknown as Response; + } + return Response.json({ profiles: [PROFILE_REFRESHED] }); + } + if (url.endsWith("/api/routing-analytics")) { + return Response.json(ANALYTICS); + } + return new Response("missing", { status: 404 }); + }); + + const { container, root } = await mountPage(); + try { + expect(container.textContent).toContain("rev-abc"); + + const retry = [...container.querySelectorAll("button")] + .find(button => button.textContent?.trim() === "Retry"); + expect(retry).toBeTruthy(); + + // Start a load whose body stays pending, then complete a newer load first. + await act(async () => { retry!.click(); }); + await tick(2); + await act(async () => { retry!.click(); }); + await tick(4); + + expect(container.textContent).toContain("rev-def"); + expect(container.textContent).toContain("\"imageInput\": true"); + + await act(async () => { + releaseStale(); + await Promise.resolve(); + }); + await tick(3); + + // The delayed older response must not replace the fresher profiles/selection. + expect(container.textContent).toContain("rev-def"); + expect(container.textContent).toContain("balanced"); + expect(container.textContent).not.toContain("policy/cheap"); + expect(container.textContent).not.toContain("rev-cheap"); + } finally { + await act(async () => { + releaseStale(); + root.unmount(); + }); + } +}); + diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 1f2b55cfc..fea3ec40a 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -127,10 +127,18 @@ describe("GitHub Actions hardening", () => { // Sharding is only safe while the shards tile the suite exactly. If the // matrix and the divisor drift apart, some files stop running and CI stays // green — the worst failure available here. Pin them to each other. - const shards = (ci.jobs?.test as { strategy?: { matrix?: { shard?: number[] } } }) + const linuxShards = (ci.jobs?.test as { strategy?: { matrix?: { shard?: number[] } } }) ?.strategy?.matrix?.shard ?? []; - expect(shards).toEqual([1, 2, 3, 4]); - expect(workflow).toContain(`--shard=\${{ matrix.shard }}/${shards.length}`); + expect(linuxShards).toEqual([1, 2, 3, 4]); + expect(workflow).toContain(`--shard=\${{ matrix.shard }}/${linuxShards.length}`); + + // Windows uses the same shard matrix after the single-leg isolate budget was + // replaced. Keep the two matrices equal so a future edit cannot reintroduce + // a partial Windows suite while Linux stays fully tiled. + const windowsShards = (ci.jobs?.["platform-windows"] as { + strategy?: { matrix?: { shard?: number[] } }; + })?.strategy?.matrix?.shard ?? []; + expect(windowsShards).toEqual(linuxShards); // The aggregate gate is the check a human trusts. Three ways to break it // silently: drop `if: always()` so it skips (and a skipped job reports @@ -168,8 +176,7 @@ describe("GitHub Actions hardening", () => { // the runner's disk and the suite passes against a tree that no longer // exists in git. const winSteps = (ci.jobs?.["platform-windows"] as { steps?: { if?: string; run?: string }[] })?.steps ?? []; - expect(winSteps.some(step => step.run?.includes("bun test --isolate tests"))).toBe(true); - expect(winSteps.some(step => step.run?.includes("--shard=${{ matrix.shard }}/4"))).toBe(true); + expect(winSteps.some(step => step.run?.includes(`--shard=\${{ matrix.shard }}/${windowsShards.length}`))).toBe(true); expect(winSteps.some(step => step.if === "runner.environment == 'self-hosted'" && step.run?.includes("git clean -xffd"))).toBe(true); diff --git a/tests/cli-headless-parity.test.ts b/tests/cli-headless-parity.test.ts index 0064c64a6..3761e68e1 100644 --- a/tests/cli-headless-parity.test.ts +++ b/tests/cli-headless-parity.test.ts @@ -89,6 +89,11 @@ describe("headless GUI parity CLI", () => { ["/api/logs", "ocx observe"], ["/api/config", "ocx config"], ["/api/settings", "ocx system"], + // Routing Intelligence (RI-04..RI-10): profiles + dry-run are mirrored by + // `ocx route policy`. Analytics is GUI-first for now; the same request + // history remains available through observe/index tooling. + ["/api/routing-profiles", "ocx route policy"], + ["/api/routing-analytics", "(none — GUI analytics surface; history via ocx observe/logs)"], ["/api/shadow", "ocx models"], ["/api/sidecar", "ocx agent"], ["/api/startup", "ocx system"], diff --git a/tests/routing-intelligence-ui.test.ts b/tests/routing-intelligence-ui.test.ts new file mode 100644 index 000000000..c4508f13a --- /dev/null +++ b/tests/routing-intelligence-ui.test.ts @@ -0,0 +1,89 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { hashBelongsToPage, readPageFromHash, resolveAppHashChange, VALID_PAGES } from "../gui/src/app-routing"; +import { + sanitizeLogEntryRouteDecision, + validCachedRouteDecision, + type LogEntryBase as LogEntry, +} from "../gui/src/pages/log-route-decision"; + +const guiRoot = join(import.meta.dir, "..", "gui", "src"); + +test("routing is a first-class dashboard page with a registered hash", () => { + expect(VALID_PAGES.has("routing")).toBe(true); + expect(readPageFromHash("routing")).toBe("routing"); + expect(hashBelongsToPage("routing", "routing")).toBe(true); + expect(resolveAppHashChange("routing").replaceTo).toBeNull(); +}); + +test("Routing page wires profiles, dry-run, and analytics against management APIs", () => { + const page = readFileSync(join(guiRoot, "pages", "RoutingProfiles.tsx"), "utf8"); + expect(page).toContain("/api/routing-profiles"); + expect(page).toContain("/api/routing-analytics"); + expect(page).toContain("/api/routing-profiles/dry-run"); + expect(page).toContain("if (!response.ok)"); + expect(page).toContain('data-page="routing"'); +}); + +test("Logs detail renders a route-decision section with an honest empty state", () => { + const page = readFileSync(join(guiRoot, "pages", "Logs.tsx"), "utf8"); + expect(page).toContain("routeDecision"); + expect(page).toContain('logs.detail.route.section'); + expect(page).toContain('logs.detail.route.unknown'); + expect(page).toContain("log-detail-route"); +}); + +const baseLog = { + timestamp: 1, + model: "gpt-test", + provider: "openai", + status: 200, + durationMs: 10, +} satisfies LogEntry; + +test("validCachedRouteDecision accepts a well-formed route decision", () => { + expect(validCachedRouteDecision(undefined)).toBe(true); + expect(validCachedRouteDecision({ + routeKind: "combo", + profile: { id: "balanced", revision: "rev-1" }, + selected: { provider: "openai", model: "gpt-5", reason: "score" }, + candidates: [{ provider: "openai", model: "gpt-5", eligible: true, exclusions: [{ code: "cooldown" }] }], + })).toBe(true); +}); + +test("validCachedRouteDecision rejects malformed candidates and nested fields", () => { + expect(validCachedRouteDecision({ candidates: "nope" as unknown as [] })).toBe(false); + expect(validCachedRouteDecision({ candidates: [{ provider: "openai", eligible: "yes" as unknown as boolean }] })).toBe(false); + expect(validCachedRouteDecision({ profile: { id: 12 as unknown as string } })).toBe(false); + expect(validCachedRouteDecision({ routeKind: 7 as unknown as string })).toBe(false); + expect(validCachedRouteDecision({ selected: { provider: 1 as unknown as string } })).toBe(false); +}); + +test("sanitizeLogEntryRouteDecision drops invalid routeDecision and keeps valid ones", () => { + const valid: LogEntry = { + ...baseLog, + routeDecision: { + routeKind: "native", + selected: { provider: "openai", model: "gpt-5" }, + }, + }; + const invalid: LogEntry = { + ...baseLog, + routeDecision: { + candidates: { provider: "openai" } as unknown as [], + }, + }; + + expect(sanitizeLogEntryRouteDecision(valid).routeDecision).toEqual(valid.routeDecision); + expect(sanitizeLogEntryRouteDecision(invalid).routeDecision).toBeUndefined(); + expect(sanitizeLogEntryRouteDecision(baseLog).routeDecision).toBeUndefined(); +}); + +test("App mounts RoutingProfiles from the sidebar NAV entry", () => { + const app = readFileSync(join(guiRoot, "App.tsx"), "utf8"); + expect(app).toContain('id: "routing"'); + expect(app).toContain("RoutingProfiles"); + expect(app).toContain('page === "routing"'); + expect(app).toContain("IconRoute"); +});