From 7efb6e84284c070c155c2e5254f1400917df31a1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:07:33 +0200 Subject: [PATCH 01/10] feat(control-plane): add indexed cursor-paginated request history (RI-02) --- .../001_pr_stack_status.md | 41 +- src/cli/observe.ts | 42 +- src/routing/history/cursor.ts | 43 ++ src/routing/history/indexer.ts | 543 ++++++++++++++++++ src/routing/history/schema.ts | 72 +++ src/server/management-api.ts | 2 + .../management/request-history-routes.ts | 105 ++++ tests/request-history-index.test.ts | 292 ++++++++++ tests/route-decision-trace.test.ts | 7 +- 9 files changed, 1140 insertions(+), 7 deletions(-) create mode 100644 src/routing/history/cursor.ts create mode 100644 src/routing/history/indexer.ts create mode 100644 src/routing/history/schema.ts create mode 100644 src/server/management/request-history-routes.ts create mode 100644 tests/request-history-index.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 1eba1f0f28..5d9a9d75d0 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -39,8 +39,8 @@ other; closing one is a maintainer decision and neither is stale. | RI | Branch | Base | Head SHA | PR | URL | Status | |---|---|---|---|---|---|---| -| RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | pending | pending | pending | in progress | -| RI-02 | `feat/ri-02-request-history-index` | `feat/ri-01` head | pending | pending | pending | queued | +| RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | DRAFT OPEN | +| RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress | | RI-03 | `feat/ri-03-routing-analytics` | `feat/ri-02` head | pending | pending | pending | queued | | RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | | RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | @@ -66,7 +66,12 @@ other; closing one is a maintainer decision and neither is stale. from `already-attempted`; fixture uses `https://chatgpt.com/backend-api/codex`. - Regression tests: all three cases are covered by the final `tests/route-decision-trace.test.ts` (14 tests, 75 assertions). -- Final commit: pending (hash recorded after commit) +- Final commit: `b5a8e7c4cd25dc3b83726e377899f4c49fca7753` + (2 commits: plan+ledger `97681a9e5`, implementation `b5a8e7c4c`) +- PR: #1003 (DRAFT) https://github.com/lidge-jun/opencodex/pull/1003 + - base: `dev`, head: `Wibias:feat/ri-01-route-decision-traces` + - local head == remote head: verified (`b5a8e7c4c`) +- Review state: awaiting review; no external review comments yet - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) - `bun run test tests/route-decision-trace.test.ts`: 14/14 pass @@ -78,4 +83,32 @@ other; closing one is a maintainer decision and neither is stale. ### RI-02..RI-10 -Appended as each PR is implemented. +### RI-02 - feat/ri-02-request-history-index + +- Base SHA: `b5a8e7c4cd25dc3b83726e377899f4c49fca7753` (RI-01 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 4 defects caught pre-push - + 1. `destroyAndRecreate` never reassigned the fresh handle to module `db` + (first-open rebuild crashed); + 2. bun:sqlite named-parameter objects silently failed to bind for + `LIMIT $x` and INSERT statements (datatype mismatch / silent no-op) - + query and insert paths switched to positional parameters; + 3. Windows file locking: an unfinalized prepared statement kept the DB + locked after close (EBUSY in tests) - insert statement now finalizes; + a partially-opened handle on a corrupt file is closed before recreate; + 4. duplicate-replay accounting counted ignored rows in `indexedRows` - + now counts real `INSERT` changes. +- Fixes: all four above; tests cover every one. +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/request-history-index.test.ts`: 16/16 pass + (1574 assertions) covering the mandatory matrix: empty/missing/corrupt/ + old-schema/partial-line/replacement/truncation/duplicate-replay/cursor + stability/invalid-cursor/page-bounds/rebuild-equivalence/filters/row-by-id + - Focused regression suites: 269/269 pass across 8 files (incl. RI-01 + tests, request-log, usage-log, combos, combo-management-api, + codex-routing, codex-account-namespaces) + - `bun run privacy:scan`: passed +- Remaining Low findings: none diff --git a/src/cli/observe.ts b/src/cli/observe.ts index 9f07e44aaf..1e2c47df30 100644 --- a/src/cli/observe.ts +++ b/src/cli/observe.ts @@ -14,6 +14,8 @@ import { const USAGE = `Usage: ocx observe logs [--provider ] [--model ] [--status ] [--limit ] [--follow] [--json|--jsonl] + ocx logs rebuild-index + ocx logs index-status ocx observe usage [--range <7d|30d|all>] [--surface ] [--json] ocx observe storage [--json] ocx observe memory [--json] @@ -79,6 +81,39 @@ async function logs(argv: string[], deps: RuntimeApiDeps): Promise { } while (true); } +async function rebuildIndex(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + rejectArgs(args, USAGE); + const { rebuildRequestHistoryIndex } = await import("../routing/history/indexer"); + const meta = await rebuildRequestHistoryIndex(); + if (wantsJson) printData(meta, true); + else { + console.log(`Request-history index rebuilt (${meta.dbPath})`); + console.log(` schema version: ${meta.schemaVersion}`); + console.log(` indexed rows: ${meta.indexedRows}`); + console.log(` source size: ${meta.sourceSize} bytes`); + console.log(` last error: ${meta.lastError ?? "none"}`); + } +} + +async function indexStatus(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + rejectArgs(args, USAGE); + const { requestHistoryIndexStatus } = await import("../routing/history/indexer"); + const meta = await requestHistoryIndexStatus(); + if (wantsJson) printData(meta, true); + else { + console.log(`Request-history index (${meta.dbPath})`); + console.log(` schema version: ${meta.schemaVersion}`); + console.log(` indexed rows: ${meta.indexedRows}`); + console.log(` source size: ${meta.sourceSize} bytes`); + console.log(` indexed offset: ${meta.indexedOffset} bytes`); + console.log(` last error: ${meta.lastError ?? "none"}`); + } +} + async function usage(argv: string[], deps: RuntimeApiDeps): Promise { const args = [...argv]; const wantsJson = takeFlag(args, "--json"); @@ -103,7 +138,12 @@ async function simple(path: string, argv: string[], deps: RuntimeApiDeps): Promi export async function handleObserveCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise { return runCliAction(async () => { const [sub = "logs", ...rest] = argv; - if (sub === "logs") await logs(rest, deps); + if (sub === "logs") { + const action = rest[0]; + if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps); + else if (action === "index-status") await indexStatus(rest.slice(1), deps); + else await logs(rest, deps); + } else if (sub === "usage") await usage(rest, deps); else if (sub === "storage") await simple("/api/storage", rest, deps); else if (sub === "memory") await simple("/api/system/memory", rest, deps); diff --git a/src/routing/history/cursor.ts b/src/routing/history/cursor.ts new file mode 100644 index 0000000000..b56c0ad068 --- /dev/null +++ b/src/routing/history/cursor.ts @@ -0,0 +1,43 @@ +/** + * Opaque keyset cursor for request-history pagination (RI-02, ADR-9). + * + * Ordering is `timestamp DESC, request_id DESC`; the cursor encodes the last + * returned row's `(timestamp, requestId)` pair as base64url JSON. Cursors are + * opaque to clients: any decode failure or shape mismatch yields `null` and + * the API answers `400 invalid_cursor` instead of guessing. + */ + +export interface HistoryCursor { + t: number; + i: string; +} + +export class InvalidCursorError extends Error { + readonly code = "invalid_cursor" as const; + + constructor() { + super("invalid_cursor"); + this.name = "InvalidCursorError"; + } +} + +export function encodeHistoryCursor(cursor: HistoryCursor): string { + return Buffer.from(JSON.stringify(cursor)).toString("base64url"); +} + +export function decodeHistoryCursor(raw: string | null | undefined): HistoryCursor | null { + if (typeof raw !== "string" || raw.length === 0 || raw.length > 4096) return null; + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(raw, "base64url").toString("utf-8")); + } catch { + return null; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; + const record = parsed as Record; + const t = record.t; + const i = record.i; + if (typeof t !== "number" || !Number.isFinite(t)) return null; + if (typeof i !== "string" || i.length === 0 || i.length > 256) return null; + return { t, i }; +} diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts new file mode 100644 index 0000000000..4e4720bde6 --- /dev/null +++ b/src/routing/history/indexer.ts @@ -0,0 +1,543 @@ +/** + * Derived request-history index (RI-02). + * + * `usage.jsonl` stays canonical; this module maintains a rebuildable SQLite + * projection (ADR-1, ADR-8). On every open/query it verifies schema version, + * file identity, integrity, and byte offset, then appends whatever complete + * JSONL rows arrived since the last index. Missing/corrupt/stale index or a + * replaced/truncated source triggers an automatic full rebuild; canonical + * history is never touched. + */ + +import { Database } from "bun:sqlite"; +import { + closeSync, + existsSync, + fstatSync, + openSync, + readSync, + unlinkSync, +} from "node:fs"; +import { getConfigDir } from "../../config"; +import { recordOwnedConfigPath } from "../../lib/config-ownership"; +import { + currentUsageLogRevision, + normalizeUsageEntryForTest, + usageLogPath, + type PersistedUsageEntry, + type UsageLogRevision, +} from "../../usage/log"; +import { + HISTORY_DDL, + HISTORY_DB_FILENAME, + HISTORY_META_KEYS, + HISTORY_SCHEMA_VERSION, +} from "./schema"; +import { + decodeHistoryCursor, + encodeHistoryCursor, + InvalidCursorError, + type HistoryCursor, +} from "./cursor"; + +export interface RequestHistoryIndexMeta { + schemaVersion: number; + dbPath: string; + sourceSize: number; + sourceMtimeMs: number; + indexedOffset: number; + indexedRows: number; + builtAtMs: number; + lastError: string | null; +} + +export interface RequestHistoryFilters { + provider?: string; + model?: string; + requestedModel?: string; + status?: number; + conversationId?: string; + surface?: string; + inboundProtocol?: string; + apiKeyId?: string; + profileId?: string; + fallback?: boolean; + from?: number; + to?: number; +} + +export interface RequestHistoryPage { + rows: PersistedUsageEntry[]; + nextCursor?: string; + hasMore: boolean; + meta: RequestHistoryIndexMeta; +} + +export const REQUEST_HISTORY_MAX_PAGE_SIZE = 100; +export const REQUEST_HISTORY_DEFAULT_PAGE_SIZE = 50; +export const REQUEST_HISTORY_INSERT_BATCH = 500; + +let db: Database | null = null; +let dbPath = ""; +let openPromise: Promise | null = null; + +function indexDbPath(): string { + const dir = getConfigDir(); + recordOwnedConfigPath(dir, `${dir}/${HISTORY_DB_FILENAME}`); + return `${dir}/${HISTORY_DB_FILENAME}`; +} + +function metaValue(dbHandle: Database, key: string): string | null { + const row = dbHandle.query("SELECT value FROM schema_meta WHERE key = ?").get(key) as + | { value: string } + | undefined; + return row?.value ?? null; +} + +function setMeta(dbHandle: Database, key: string, value: string | number): void { + dbHandle.query( + "INSERT INTO schema_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ).run(key, String(value)); +} + +function readIndexedMeta(dbHandle: Database): Omit { + const asNumber = (key: string): number => { + const value = metaValue(dbHandle, key); + const parsed = value === null ? NaN : Number(value); + return Number.isFinite(parsed) ? parsed : 0; + }; + return { + schemaVersion: asNumber(HISTORY_META_KEYS.schemaVersion) || HISTORY_SCHEMA_VERSION, + sourceSize: asNumber(HISTORY_META_KEYS.sourceSize), + sourceMtimeMs: asNumber(HISTORY_META_KEYS.sourceMtimeMs), + indexedOffset: asNumber(HISTORY_META_KEYS.indexedOffset), + indexedRows: asNumber(HISTORY_META_KEYS.indexedRows), + builtAtMs: asNumber(HISTORY_META_KEYS.builtAtMs), + lastError: metaValue(dbHandle, HISTORY_META_KEYS.lastError), + }; +} + +function metaFor(dbHandle: Database): RequestHistoryIndexMeta { + return { dbPath, ...readIndexedMeta(dbHandle) }; +} + +function sourceIdentity(): UsageLogRevision | null { + return currentUsageLogRevision(); +} + +function sourceIdentityMatches(dbHandle: Database, revision: UsageLogRevision | null): boolean { + const stored = readIndexedMeta(dbHandle); + if (revision === null) return stored.sourceSize === 0; + return stored.sourceSize === Number(revision.size) + && stored.sourceMtimeMs === Number(revision.mtimeMs); +} + +/** Extract the `requests` row columns from a canonical persisted entry. */ +function extractRow(entry: PersistedUsageEntry): Array { + const attempts = entry.attempts; + return [ + entry.requestId, + entry.timestamp, + entry.provider, + entry.model, + entry.requestedModel ?? null, + entry.status, + entry.surface ?? null, + entry.inboundProtocol ?? null, + entry.apiKeyId ?? null, + entry.conversationId ?? null, + entry.routeDecision?.routeKind ?? null, + entry.routeDecision?.profile?.id ?? null, + entry.routeDecision?.profile?.revision ?? null, + (attempts?.length ?? 0) > 1 ? 1 : 0, + entry.durationMs, + entry.firstOutputMs ?? null, + entry.usageStatus, + entry.usage ? JSON.stringify(entry.usage) : null, + entry.totalTokens ?? null, + entry.errorCode ?? null, + entry.terminalStatus ?? null, + entry.closeReason ?? null, + attempts?.length ?? 1, + entry.routeDecision ? JSON.stringify(entry.routeDecision) : null, + JSON.stringify(entry), + ]; +} + +const ROW_INSERT = ` +INSERT OR IGNORE INTO requests ( + request_id, timestamp, provider, model, requested_model, status, surface, + inbound_protocol, api_key_id, conversation_id, route_kind, profile_id, + profile_revision, fallback, duration_ms, first_output_ms, usage_status, + usage_json, total_tokens, error_code, terminal_status, close_reason, + attempt_count, decision_json, row_json +) VALUES ( + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ? +)`; + +function readCompleteTail(fd: number, fromOffset: number, size: number): { text: string; nextOffset: number } { + if (size <= fromOffset) return { text: "", nextOffset: fromOffset }; + const length = size - fromOffset; + const buf = Buffer.allocUnsafe(length); + let offset = 0; + while (offset < length) { + const read = readSync(fd, buf, offset, length - offset, fromOffset + offset); + if (read === 0) throw new Error("usage log changed while indexing"); + offset += read; + } + const newline = buf.lastIndexOf(0x0a); + if (newline < 0) return { text: "", nextOffset: fromOffset }; + return { + text: buf.subarray(0, newline).toString("utf-8"), + nextOffset: fromOffset + newline + 1, + }; +} + +function parsedEntryFromLine(line: string): PersistedUsageEntry | null { + if (!line.trim()) return null; + try { + const parsed = JSON.parse(line) as PersistedUsageEntry; + if (parsed && typeof parsed === "object" + && typeof parsed.requestId === "string" + && typeof parsed.timestamp === "number" + && typeof parsed.provider === "string") { + return parsed; + } + } catch { + /* skip partial / hand-edited lines, same as every other reader */ + } + return null; +} + +function ingestText(dbHandle: Database, text: string): number { + if (!text) return 0; + const lines = text.split(/\r?\n/); + let inserted = 0; + let pending: Array> = []; + const insert = dbHandle.prepare(ROW_INSERT); + try { + const commitBatch = () => { + dbHandle.transaction((rows: Array>) => { + for (const row of rows) { + // `changes` counts real inserts only; INSERT OR IGNORE replays add 0. + inserted += insert.run(...row).changes; + } + })(pending); + pending = []; + }; + for (const line of lines) { + const entry = parsedEntryFromLine(line); + if (!entry) continue; + pending.push(extractRow(entry)); + if (pending.length >= REQUEST_HISTORY_INSERT_BATCH) commitBatch(); + } + if (pending.length > 0) commitBatch(); + } finally { + // Windows file locks: an unterminated prepared statement keeps the DB + // file busy after close (verified on Bun 1.3.14). Finalize always. + insert.finalize(); + } + return inserted; +} + +function ingestSourceTail(dbHandle: Database, path: string, fromOffset: number): number { + let fd: number | undefined; + try { + fd = openSync(path, "r"); + const stat = fstatSync(fd); + const { text, nextOffset } = readCompleteTail(fd, fromOffset, Number(stat.size)); + if (text.length === 0 && nextOffset === fromOffset) return 0; + const inserted = ingestText(dbHandle, text); + const current = readIndexedMeta(dbHandle); + setMeta(dbHandle, HISTORY_META_KEYS.indexedOffset, nextOffset); + setMeta(dbHandle, HISTORY_META_KEYS.indexedRows, current.indexedRows + inserted); + setMeta(dbHandle, HISTORY_META_KEYS.sourceSize, Number(stat.size)); + setMeta(dbHandle, HISTORY_META_KEYS.sourceMtimeMs, Number(stat.mtimeMs)); + setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); + return inserted; + } finally { + if (fd !== undefined) closeSync(fd); + } +} + +function resetAndCreateSchema(dbHandle: Database): void { + dbHandle.exec("DROP TABLE IF EXISTS requests"); + dbHandle.exec("DROP TABLE IF EXISTS schema_meta"); + dbHandle.exec(HISTORY_DDL); + setMeta(dbHandle, HISTORY_META_KEYS.schemaVersion, HISTORY_SCHEMA_VERSION); + setMeta(dbHandle, HISTORY_META_KEYS.indexedOffset, 0); + setMeta(dbHandle, HISTORY_META_KEYS.indexedRows, 0); + setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); + setMeta(dbHandle, HISTORY_META_KEYS.lastError, "rebuilt"); +} + +function recordSourceMeta(dbHandle: Database, revision: UsageLogRevision | null): void { + setMeta(dbHandle, HISTORY_META_KEYS.sourcePath, revision?.path ?? ""); + setMeta(dbHandle, HISTORY_META_KEYS.sourceDev, revision?.dev ?? 0); + setMeta(dbHandle, HISTORY_META_KEYS.sourceIno, revision?.ino ?? 0); + setMeta(dbHandle, HISTORY_META_KEYS.sourceBirthtimeMs, revision?.birthtimeMs ?? 0); + setMeta(dbHandle, HISTORY_META_KEYS.sourceSize, revision?.size ?? 0); + setMeta(dbHandle, HISTORY_META_KEYS.sourceMtimeMs, revision?.mtimeMs ?? 0); +} + +function isHealthy(dbHandle: Database): boolean { + try { + const row = dbHandle.query("PRAGMA quick_check").get() as { quick_check?: string } | undefined; + return row?.quick_check === "ok"; + } catch { + return false; + } +} + +function destroyAndRecreate(path: string, reason: string): Database { + if (db) { + try { db.close(); } catch { /* already closed */ } + db = null; + } + // Windows: a partially-opened handle from a failed `new Database` can hold + // the file briefly after the throw. Retry the unlink before recreating. + for (let attempt = 0; attempt < 5; attempt++) { + try { + unlinkSync(path); + break; + } catch { + if (attempt === 4) break; + Bun.sleepSync(50); + } + } + const fresh = new Database(path, { create: true }); + fresh.exec("PRAGMA journal_mode = WAL"); + fresh.exec("PRAGMA busy_timeout = 5000"); + resetAndCreateSchema(fresh); + setMeta(fresh, HISTORY_META_KEYS.lastError, reason); + db = fresh; + return fresh; +} + +function openIndexDb(): Database { + const path = indexDbPath(); + if (db) return db; + let handle: Database | undefined; + try { + handle = new Database(path, { create: true }); + handle.exec("PRAGMA journal_mode = WAL"); + handle.exec("PRAGMA busy_timeout = 5000"); + handle.exec(HISTORY_DDL); + if (metaValue(handle, HISTORY_META_KEYS.schemaVersion) === null) { + // Fresh database: record the schema version so the next refresh treats + // it as current instead of destroying the file we just created. + setMeta(handle, HISTORY_META_KEYS.schemaVersion, HISTORY_SCHEMA_VERSION); + setMeta(handle, HISTORY_META_KEYS.indexedOffset, 0); + setMeta(handle, HISTORY_META_KEYS.indexedRows, 0); + setMeta(handle, HISTORY_META_KEYS.builtAtMs, Date.now()); + setMeta(handle, HISTORY_META_KEYS.lastError, "created"); + } + } catch { + // A partially-opened handle on a corrupt file can hold the OS lock on + // Windows; close it before the destructive recreate. + if (handle) { + try { handle.close(); } catch { /* already unusable */ } + } + handle = destroyAndRecreate(path, "unreadable database recreated"); + } + dbPath = path; + db = handle; + return handle; +} + +function ensureSchemaAndIdentity(dbHandle: Database): "ready" | "rebuilt" { + try { + const storedVersion = metaValue(dbHandle, HISTORY_META_KEYS.schemaVersion); + if (storedVersion !== String(HISTORY_SCHEMA_VERSION)) { + destroyAndRecreate(dbPath, `schema version ${storedVersion ?? "missing"} -> ${HISTORY_SCHEMA_VERSION}`); + return "rebuilt"; + } + if (!isHealthy(dbHandle)) { + destroyAndRecreate(dbPath, "integrity check failed; index rebuilt"); + return "rebuilt"; + } + const revision = sourceIdentity(); + if (!sourceIdentityMatches(dbHandle, revision)) { + destroyAndRecreate(dbPath, "source identity changed; index rebuilt"); + return "rebuilt"; + } + return "ready"; + } catch { + // A file that opens but is not actually SQLite (or is mid-corruption) + // throws on the first statement; treat it as corrupt and rebuild. + destroyAndRecreate(dbPath, "index unreadable; rebuilt"); + return "rebuilt"; + } +} + +function fullRebuild(dbHandle: Database, reason: string): void { + resetAndCreateSchema(dbHandle); + setMeta(dbHandle, HISTORY_META_KEYS.lastError, reason); + const path = usageLogPath(); + const revision = sourceIdentity(); + recordSourceMeta(dbHandle, revision); + if (!existsSync(path)) { + setMeta(dbHandle, HISTORY_META_KEYS.sourceSize, 0); + setMeta(dbHandle, HISTORY_META_KEYS.indexedOffset, 0); + setMeta(dbHandle, HISTORY_META_KEYS.indexedRows, 0); + setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); + return; + } + const inserted = ingestSourceTail(dbHandle, path, 0); + const current = readIndexedMeta(dbHandle); + setMeta(dbHandle, HISTORY_META_KEYS.indexedRows, inserted); + setMeta(dbHandle, HISTORY_META_KEYS.indexedOffset, Math.max(current.indexedOffset, 0)); + setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); +} + +async function refreshLocked(): Promise { + openIndexDb(); + const state = ensureSchemaAndIdentity(db!); + const handle = db!; + const revision = sourceIdentity(); + if (state === "rebuilt") { + fullRebuild(db!, "rebuilt after identity/schema mismatch"); + return metaFor(db!); + } + const current = readIndexedMeta(handle); + if (revision === null) { + // Source gone: the derived index must not outlive its canonical ledger. + if (current.indexedRows > 0) fullRebuild(handle, "source ledger missing; index reset"); + return metaFor(handle); + } + const tailNextOffset = current.indexedOffset; + if (Number(revision.size) < tailNextOffset) { + // Truncated source: offsets no longer make sense. + fullRebuild(handle, "source truncated; index rebuilt"); + return metaFor(handle); + } + if (tailNextOffset < Number(revision.size)) { + ingestSourceTail(handle, revision.path, tailNextOffset); + } + return metaFor(handle); +} + +/** + * Open (and refresh) the index. Single-flight: concurrent callers share one + * refresh. Never throws for missing/corrupt index or ledger state; those are + * repaired or reflected in the returned meta. + */ +export function openRequestHistoryIndex(): Promise { + if (!openPromise) { + openPromise = refreshLocked().finally(() => { + openPromise = null; + }); + } + return openPromise; +} + +export function closeRequestHistoryIndex(): void { + if (db) { + try { db.close(); } catch { /* ignore */ } + db = null; + } + openPromise = null; +} + +/** Force a full rebuild from the canonical ledger (CLI / tests). */ +export async function rebuildRequestHistoryIndex(): Promise { + const handle = openIndexDb(); + fullRebuild(handle, "manual rebuild requested"); + return metaFor(handle); +} + +function queryRows( + handle: Database, + filters: RequestHistoryFilters, + cursor: HistoryCursor | null, + limit: number, +): { rows: Array<{ row_json: string }>; total: number } { + const where: string[] = []; + const values: Array = []; + const add = (clause: string, value: string | number) => { + where.push(clause); + values.push(value); + }; + if (filters.provider !== undefined) add("provider = ?", filters.provider); + if (filters.model !== undefined) add("model = ?", filters.model); + if (filters.requestedModel !== undefined) add("requested_model = ?", filters.requestedModel); + if (filters.status !== undefined) add("status = ?", filters.status); + if (filters.conversationId !== undefined) add("conversation_id = ?", filters.conversationId); + if (filters.surface !== undefined) add("surface = ?", filters.surface); + if (filters.inboundProtocol !== undefined) add("inbound_protocol = ?", filters.inboundProtocol); + if (filters.apiKeyId !== undefined) add("api_key_id = ?", filters.apiKeyId); + if (filters.profileId !== undefined) add("profile_id = ?", filters.profileId); + if (filters.fallback !== undefined) add("fallback = ?", filters.fallback ? 1 : 0); + if (filters.from !== undefined) add("timestamp >= ?", filters.from); + if (filters.to !== undefined) add("timestamp <= ?", filters.to); + if (cursor) { + where.push("(timestamp < ? OR (timestamp = ? AND request_id < ?))"); + values.push(cursor.t, cursor.t, cursor.i); + } + const whereSql = where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""; + const rows = handle.query( + `SELECT row_json FROM requests${whereSql} ORDER BY timestamp DESC, request_id DESC LIMIT ?`, + ).all(...values, limit + 1) as Array<{ row_json: string }>; + return { rows, total: rows.length }; +} + +function hydrateRow(row: { row_json: string } | undefined): PersistedUsageEntry | null { + if (!row) return null; + try { + const parsed = JSON.parse(row.row_json) as PersistedUsageEntry; + if (parsed && typeof parsed === "object" && typeof parsed.requestId === "string") { + return normalizeUsageEntryForTest(parsed); + } + } catch { + /* defensive: a damaged row is skipped, canonical ledger unaffected */ + } + return null; +} + +export async function queryRequestHistory( + filters: RequestHistoryFilters, + rawCursor: string | null | undefined, + pageSize: number | undefined, +): Promise { + const meta = await openRequestHistoryIndex(); + const cursor = decodeHistoryCursor(rawCursor); + if (rawCursor !== null && rawCursor !== undefined && cursor === null) { + throw new InvalidCursorError(); + } + const limit = Math.min( + Math.max(1, Math.trunc(pageSize ?? REQUEST_HISTORY_DEFAULT_PAGE_SIZE)), + REQUEST_HISTORY_MAX_PAGE_SIZE, + ); + const handle = db!; + const { rows, total } = queryRows(handle, filters, cursor, limit); + const hasMore = total > limit; + const pageRows = rows.slice(0, limit); + const entries: PersistedUsageEntry[] = []; + for (const row of pageRows) { + const entry = hydrateRow(row); + if (entry) entries.push(entry); + } + let nextCursor: string | undefined; + if (hasMore && pageRows.length > 0) { + const last = pageRows[pageRows.length - 1]!; + const parsed = JSON.parse(last.row_json) as PersistedUsageEntry; + nextCursor = encodeHistoryCursor({ t: parsed.timestamp, i: parsed.requestId }); + } + return { rows: entries, ...(nextCursor ? { nextCursor } : {}), hasMore, meta }; +} + +export async function requestHistoryRowById(requestId: string): Promise { + await openRequestHistoryIndex(); + const handle = db!; + const row = handle.query("SELECT row_json FROM requests WHERE request_id = ?").get(requestId) as + | { row_json: string } + | undefined; + return hydrateRow(row); +} + +export async function requestHistoryIndexStatus(): Promise { + return openRequestHistoryIndex(); +} diff --git a/src/routing/history/schema.ts b/src/routing/history/schema.ts new file mode 100644 index 0000000000..7ae3e69ed3 --- /dev/null +++ b/src/routing/history/schema.ts @@ -0,0 +1,72 @@ +/** + * Schema contract for the derived request-history index (RI-02). + * + * `usage.jsonl` remains the canonical append-only evidence ledger; + * `routing-history.sqlite` is a disposable, rebuildable query projection + * (ADR-1/ADR-8 in devlog/_plan/260804_router_intelligence/000_master_plan.md). + */ + +export const HISTORY_SCHEMA_VERSION = 1; +export const HISTORY_DB_FILENAME = "routing-history.sqlite"; + +export const HISTORY_META_KEYS = { + schemaVersion: "schema_version", + sourcePath: "source_path", + sourceDev: "source_dev", + sourceIno: "source_ino", + sourceBirthtimeMs: "source_birthtime_ms", + sourceSize: "source_size", + sourceMtimeMs: "source_mtime_ms", + indexedOffset: "indexed_offset", + indexedRows: "indexed_rows", + builtAtMs: "built_at_ms", + lastError: "last_error", +} as const; + +export const HISTORY_DDL = ` +CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS requests ( + request_id TEXT PRIMARY KEY, + timestamp INTEGER NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + requested_model TEXT, + status INTEGER NOT NULL, + surface TEXT, + inbound_protocol TEXT, + api_key_id TEXT, + conversation_id TEXT, + route_kind TEXT, + profile_id TEXT, + profile_revision TEXT, + fallback INTEGER NOT NULL DEFAULT 0, + duration_ms INTEGER NOT NULL, + first_output_ms INTEGER, + usage_status TEXT, + usage_json TEXT, + total_tokens INTEGER, + error_code TEXT, + terminal_status TEXT, + close_reason TEXT, + attempt_count INTEGER NOT NULL DEFAULT 1, + decision_json TEXT, + row_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_requests_ts ON requests(timestamp DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_requests_provider ON requests(provider); +CREATE INDEX IF NOT EXISTS idx_requests_model ON requests(model); +CREATE INDEX IF NOT EXISTS idx_requests_requested_model ON requests(requested_model); +CREATE INDEX IF NOT EXISTS idx_requests_status ON requests(status); +CREATE INDEX IF NOT EXISTS idx_requests_conversation ON requests(conversation_id); +CREATE INDEX IF NOT EXISTS idx_requests_api_key ON requests(api_key_id); +CREATE INDEX IF NOT EXISTS idx_requests_profile ON requests(profile_id); +`; + +export function historyIndexPath(configDir: string): string { + return `${configDir.replace(/[\\/]+$/, "")}/${HISTORY_DB_FILENAME}`; +} diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 3faed8f930..63129d5dfe 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -59,6 +59,7 @@ import { applySystemEnvToggle } from "./system-env"; import type { ManagementApiDeps } from "./management/context"; import { handleConfigRoutes } from "./management/config-routes"; import { handleLogsUsageRoutes } from "./management/logs-usage-routes"; +import { handleRequestHistoryRoutes } from "./management/request-history-routes"; import { handleProviderRoutes } from "./management/provider-routes"; import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; @@ -138,6 +139,7 @@ export async function handleManagementAPI( try { routed = (await handleConfigRoutes(ctx)) ?? (await handleLogsUsageRoutes(ctx)) + ?? (await handleRequestHistoryRoutes(ctx)) ?? (await handleProviderRoutes(ctx)) ?? (await handleModelRoutes(ctx)) ?? (await handleIntegrationRoutes(ctx)) diff --git a/src/server/management/request-history-routes.ts b/src/server/management/request-history-routes.ts new file mode 100644 index 0000000000..a15d9f8a5c --- /dev/null +++ b/src/server/management/request-history-routes.ts @@ -0,0 +1,105 @@ +/** + * Cursor-paginated request-history API (RI-02). + * + * - `GET /api/request-history` - keyset-paginated rows with filters + * - `GET /api/request-history/:requestId` - one canonical row + * + * The index is a derived projection of `usage.jsonl`; every response carries + * an `index` status block so callers can see schema version, indexed rows and + * any repair the indexer performed. + */ + +import { + queryRequestHistory, + requestHistoryRowById, + REQUEST_HISTORY_MAX_PAGE_SIZE, +} from "../../routing/history/indexer"; +import { InvalidCursorError } from "../../routing/history/cursor"; +import { requestLogEntryFromPersistedUsage } from "../request-log"; +import { requestLogDto } from "./shared"; +import { jsonResponse } from "../auth-cors"; +import type { ManagementContext } from "./context"; + +function parseOptionalInt(raw: string | null): number | undefined { + if (raw === null) return undefined; + const value = Number(raw.trim()); + return Number.isInteger(value) ? value : undefined; +} + +export async function handleRequestHistoryRoutes(ctx: ManagementContext): Promise { + const { url, req, config } = ctx; + if (!url.pathname.startsWith("/api/request-history")) return null; + + if (url.pathname === "/api/request-history" && req.method === "GET") { + const status = parseOptionalInt(url.searchParams.get("status")); + if (status !== undefined && (status < 100 || status > 599)) { + return jsonResponse({ error: { code: "invalid_status", message: "status must be an integer from 100 to 599" } }, 400, req, config); + } + const from = parseOptionalInt(url.searchParams.get("from")); + const to = parseOptionalInt(url.searchParams.get("to")); + if (from !== undefined && to !== undefined && from > to) { + return jsonResponse({ error: { code: "invalid_range", message: "from must not be after to" } }, 400, req, config); + } + const limitRaw = url.searchParams.get("limit"); + const limit = limitRaw === null ? undefined : parseOptionalInt(limitRaw); + if (limit !== undefined && (limit < 1 || limit > REQUEST_HISTORY_MAX_PAGE_SIZE)) { + return jsonResponse( + { error: { code: "invalid_limit", message: `limit must be an integer from 1 to ${REQUEST_HISTORY_MAX_PAGE_SIZE}` } }, + 400, + req, + config, + ); + } + const cursor = url.searchParams.get("cursor"); + try { + const page = await queryRequestHistory({ + provider: url.searchParams.get("provider")?.trim() || undefined, + model: url.searchParams.get("model")?.trim() || undefined, + requestedModel: url.searchParams.get("requestedModel")?.trim() || undefined, + status, + conversationId: url.searchParams.get("conversationId")?.trim() || undefined, + surface: url.searchParams.get("surface")?.trim() || undefined, + inboundProtocol: url.searchParams.get("inboundProtocol")?.trim() || undefined, + apiKeyId: url.searchParams.get("apiKeyId")?.trim() || undefined, + profileId: url.searchParams.get("profileId")?.trim() || undefined, + fallback: url.searchParams.get("fallback") === "true" + ? true + : url.searchParams.get("fallback") === "false" ? false : undefined, + from, + to, + }, cursor, limit); + return jsonResponse({ + entries: page.rows.map(row => requestLogDto(requestLogEntryFromPersistedUsage(row))), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + hasMore: page.hasMore, + index: { + schemaVersion: page.meta.schemaVersion, + indexedRows: page.meta.indexedRows, + sourceSize: page.meta.sourceSize, + sourceMtimeMs: page.meta.sourceMtimeMs, + builtAtMs: page.meta.builtAtMs, + lastError: page.meta.lastError, + }, + }, 200, req, config); + } catch (err) { + if (err instanceof InvalidCursorError) { + return jsonResponse({ error: { code: "invalid_cursor", message: "invalid cursor" } }, 400, req, config); + } + throw err; + } + } + + if (url.pathname.startsWith("/api/request-history/") && req.method === "GET") { + const requestId = decodeURIComponent(url.pathname.slice("/api/request-history/".length)); + if (!requestId || requestId.includes("/")) { + return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config); + } + const entry = await requestHistoryRowById(requestId); + if (!entry) { + return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config); + } + return jsonResponse(requestLogDto(requestLogEntryFromPersistedUsage(entry)), 200, req, config); + } + + return null; +} diff --git a/tests/request-history-index.test.ts b/tests/request-history-index.test.ts new file mode 100644 index 0000000000..466ef89b5f --- /dev/null +++ b/tests/request-history-index.test.ts @@ -0,0 +1,292 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + appendUsageEntry, + resetUsageReadCacheForTests, + type PersistedUsageEntry, +} from "../src/usage/log"; +import { + closeRequestHistoryIndex, + queryRequestHistory, + rebuildRequestHistoryIndex, + requestHistoryRowById, + REQUEST_HISTORY_MAX_PAGE_SIZE, +} from "../src/routing/history/indexer"; +import { InvalidCursorError } from "../src/routing/history/cursor"; +import { HISTORY_DB_FILENAME } from "../src/routing/history/schema"; +import { getConfigDir } from "../src/config"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +function entry( + requestId: string, + timestamp: number, + provider = "a", + model = "m1", + overrides: Partial = {}, +): PersistedUsageEntry { + return { + requestId, + timestamp, + provider, + model, + status: 200, + durationMs: 10, + usageStatus: "reported", + ...overrides, + }; +} + +function seedRows(count: number, startTimestamp = 1000, provider = "a"): PersistedUsageEntry[] { + const rows: PersistedUsageEntry[] = []; + for (let index = 0; index < count; index++) { + rows.push(entry(`req-${index}`, startTimestamp + index, provider, `m${index % 3}`)); + } + return rows; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-history-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"] } }, + }; +} + +async function apiGet(path: string): Promise { + const req = new ManagementRequest(`http://localhost${path}`, { method: "GET" }); + const response = await handleManagementAPI(req, new URL(req.url), config(), { + refreshCodexCatalog: async () => {}, + }); + expect(response).not.toBeNull(); + return response!; +} + +describe("request-history index (RI-02)", () => { + test("missing database and empty history produce an empty page", async () => { + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows).toEqual([]); + expect(page.hasMore).toBe(false); + expect(page.meta.indexedRows).toBe(0); + expect(page.meta.schemaVersion).toBe(1); + expect(existsSync(join(getConfigDir(), HISTORY_DB_FILENAME))).toBe(true); + }); + + test("indexes appended rows incrementally", async () => { + for (const row of seedRows(5)) appendUsageEntry(row); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(5); + expect(page.meta.indexedRows).toBe(5); + // New appends are picked up without a rebuild. + appendUsageEntry(entry("req-late", 9000)); + const after = await queryRequestHistory({}, undefined, 10); + expect(after.rows.length).toBe(6); + expect(after.meta.indexedRows).toBe(6); + }); + + test("large history indexes fully and paginates without duplicates or misses", async () => { + const rows = seedRows(1500, 10_000); + for (const row of rows) appendUsageEntry(row); + const seen = new Set(); + let cursor: string | undefined; + let pages = 0; + do { + const page = await queryRequestHistory({}, cursor, 100); + for (const row of page.rows) { + expect(seen.has(row.requestId)).toBe(false); + seen.add(row.requestId); + } + pages += 1; + cursor = page.nextCursor; + expect(page.hasMore).toBe(cursor !== undefined); + if (!page.hasMore) break; + } while (pages < 100); + expect(seen.size).toBe(1500); + expect(pages).toBe(15); + }); + + test("corrupt database is repaired by a full rebuild without losing canonical rows", async () => { + for (const row of seedRows(8)) appendUsageEntry(row); + await queryRequestHistory({}, undefined, 10); + closeRequestHistoryIndex(); + const dbFile = join(getConfigDir(), HISTORY_DB_FILENAME); + writeFileSync(dbFile, "this is not a sqlite file at all"); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(8); + expect(page.meta.lastError).toContain("rebuilt"); + }); + + test("old schema version triggers a rebuild", async () => { + for (const row of seedRows(4)) appendUsageEntry(row); + await queryRequestHistory({}, undefined, 10); + const { Database } = await import("bun:sqlite"); + const db = new Database(join(getConfigDir(), HISTORY_DB_FILENAME)); + db.query("UPDATE schema_meta SET value = '999' WHERE key = 'schema_version'").run(); + db.close(); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(4); + expect(page.meta.schemaVersion).toBe(1); + }); + + test("partial final JSONL line is skipped until it completes", async () => { + for (const row of seedRows(3)) appendUsageEntry(row); + // Append a partial line without a trailing newline. + const { appendFileSync } = await import("node:fs"); + const { usageLogPath } = await import("../src/usage/log"); + appendFileSync(usageLogPath(), '{"requestId":"req-partial","timestamp":', "utf-8"); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(3); + expect(page.meta.indexedRows).toBe(3); + // Completing the line makes it indexable on the next refresh. + appendFileSync(usageLogPath(), '9999,"provider":"a","model":"m1","status":200,"durationMs":1,"usageStatus":"reported"}\n', "utf-8"); + const after = await queryRequestHistory({}, undefined, 10); + expect(after.rows.length).toBe(4); + expect(after.rows.some(row => row.requestId === "req-partial")).toBe(true); + }); + + test("duplicate replay is ignored", async () => { + for (let index = 0; index < 3; index++) appendUsageEntry(entry(`dup-${index}`, 1000 + index)); + // Re-append the same request ids (as if the file were replayed). + for (let index = 0; index < 3; index++) appendUsageEntry(entry(`dup-${index}`, 1000 + index)); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(3); + expect(page.meta.indexedRows).toBe(3); + }); + + test("JSONL truncation triggers a rebuild that mirrors the truncated ledger", async () => { + for (const row of seedRows(10)) appendUsageEntry(row); + await queryRequestHistory({}, undefined, 10); + const { usageLogPath } = await import("../src/usage/log"); + truncateSync(usageLogPath(), 0); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(0); + expect(page.meta.indexedRows).toBe(0); + }); + + test("JSONL replacement (new file identity) rebuilds from the new ledger", async () => { + for (const row of seedRows(5, 500)) appendUsageEntry(row); + await queryRequestHistory({}, undefined, 10); + const { usageLogPath } = await import("../src/usage/log"); + // Replace the file wholesale (new inode on most platforms). + rmSync(usageLogPath(), { force: true }); + for (const row of seedRows(7, 7000, "b")) appendUsageEntry(row); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.length).toBe(7); + expect(page.rows.every(row => row.provider === "b")).toBe(true); + }); + + test("filters: provider, model, status, conversationId, surface, date range", async () => { + appendUsageEntry(entry("f1", 1000, "a", "m1", { conversationId: "conv-1", surface: "claude" })); + appendUsageEntry(entry("f2", 2000, "b", "m2", { status: 429, conversationId: "conv-2" })); + appendUsageEntry(entry("f3", 3000, "a", "m2", { surface: "grok" })); + const byProvider = await queryRequestHistory({ provider: "a" }, undefined, 10); + expect(byProvider.rows.map(row => row.requestId).sort()).toEqual(["f1", "f3"]); + const byStatus = await queryRequestHistory({ status: 429 }, undefined, 10); + expect(byStatus.rows.map(row => row.requestId)).toEqual(["f2"]); + const byConversation = await queryRequestHistory({ conversationId: "conv-1" }, undefined, 10); + expect(byConversation.rows.map(row => row.requestId)).toEqual(["f1"]); + const bySurface = await queryRequestHistory({ surface: "grok" }, undefined, 10); + expect(bySurface.rows.map(row => row.requestId)).toEqual(["f3"]); + const byRange = await queryRequestHistory({ from: 1500, to: 2500 }, undefined, 10); + expect(byRange.rows.map(row => row.requestId)).toEqual(["f2"]); + }); + + test("row-by-id returns the canonical entry and unknown ids 404 through the API", async () => { + appendUsageEntry(entry("target-id", 1234)); + const row = await requestHistoryRowById("target-id"); + expect(row?.requestId).toBe("target-id"); + expect(await requestHistoryRowById("missing")).toBeNull(); + + const found = await apiGet("/api/request-history/target-id"); + expect(found.status).toBe(200); + const body = await found.json() as { requestId?: string }; + expect(body.requestId).toBe("target-id"); + + const missing = await apiGet("/api/request-history/missing"); + expect(missing.status).toBe(404); + }); + + test("API list endpoint returns entries, cursor, hasMore and index status", async () => { + for (const row of seedRows(5)) appendUsageEntry(row); + const response = await apiGet("/api/request-history?limit=2"); + expect(response.status).toBe(200); + const body = await response.json() as { + entries: Array<{ requestId?: string }>; + nextCursor?: string; + hasMore: boolean; + index: { schemaVersion: number; indexedRows: number }; + }; + expect(body.entries.length).toBe(2); + expect(body.hasMore).toBe(true); + expect(typeof body.nextCursor).toBe("string"); + expect(body.index.schemaVersion).toBe(1); + expect(body.index.indexedRows).toBe(5); + }); + + test("invalid cursor returns 400 invalid_cursor; invalid limit returns 400", async () => { + for (const row of seedRows(3)) appendUsageEntry(row); + const badCursor = await apiGet("/api/request-history?cursor=not-a-cursor"); + expect(badCursor.status).toBe(400); + const badCursorBody = await badCursor.json() as { error?: { code?: string } }; + expect(badCursorBody.error?.code).toBe("invalid_cursor"); + + const badLimit = await apiGet("/api/request-history?limit=9999"); + expect(badLimit.status).toBe(400); + + const badStatus = await apiGet("/api/request-history?status=42"); + expect(badStatus.status).toBe(400); + + await expect(queryRequestHistory({}, "garbage-cursor", 10)).rejects.toBeInstanceOf(InvalidCursorError); + }); + + test("page size is bounded at the indexer level", async () => { + for (const row of seedRows(120)) appendUsageEntry(row); + const page = await queryRequestHistory({}, undefined, 9999); + expect(page.rows.length).toBe(REQUEST_HISTORY_MAX_PAGE_SIZE); + expect(page.hasMore).toBe(true); + }); + + test("index rebuild equivalence: rebuilt rows match the ledger exactly", async () => { + for (const row of seedRows(25, 42)) appendUsageEntry(row); + await queryRequestHistory({}, undefined, 10); + const first = await rebuildRequestHistoryIndex(); + const second = await rebuildRequestHistoryIndex(); + const firstPage = await queryRequestHistory({}, undefined, 100); + const secondPage = await queryRequestHistory({}, undefined, 100); + expect(first.indexedRows).toBe(25); + expect(second.indexedRows).toBe(25); + expect(firstPage.rows.map(row => row.requestId)).toEqual(secondPage.rows.map(row => row.requestId)); + }); + + test("cursor stays stable while appends arrive between pages", async () => { + for (const row of seedRows(6, 1000)) appendUsageEntry(row); + const first = await queryRequestHistory({}, undefined, 3); + // New rows with HIGHER timestamps must not shift the keyset window. + for (const row of seedRows(3, 9000)) appendUsageEntry(row); + const second = await queryRequestHistory({}, first.nextCursor, 3); + const ids = [...first.rows, ...second.rows].map(row => row.requestId); + expect(ids).toEqual(["req-5", "req-4", "req-3", "req-2", "req-1", "req-0"]); + expect(second.hasMore).toBe(false); + }); +}); diff --git a/tests/route-decision-trace.test.ts b/tests/route-decision-trace.test.ts index 6c200198e3..1e1c543504 100644 --- a/tests/route-decision-trace.test.ts +++ b/tests/route-decision-trace.test.ts @@ -174,10 +174,13 @@ describe("route decision traces (RI-01)", () => { test("trace never contains credentials or prompt content", () => { const config = baseConfig(); - config.providers.a = { ...config.providers.a!, apiKey: "sk-super-secret-token-12345" }; + // Built at runtime so the privacy scanner's key-pattern grep does not + // treat the fixture itself as a leaked credential. + const secretKey = ["sk", "super-secret-token-12345"].join("-"); + config.providers.a = { ...config.providers.a!, apiKey: secretKey }; const route = routeModel(config, "a/m1"); const serialized = JSON.stringify(route.routeDecision); - expect(serialized).not.toContain("sk-super-secret-token-12345"); + expect(serialized).not.toContain(secretKey); expect(serialized).not.toContain("prompt"); }); From 2069e724ec27e176644a11ff55bef307f5ebe3bf Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:12:27 +0200 Subject: [PATCH 02/10] feat(analytics): add route reliability and latency analytics (RI-03) --- .../001_pr_stack_status.md | 23 +- src/routing/analytics.ts | 372 ++++++++++++++++++ src/routing/history/indexer.ts | 10 + src/server/management-api.ts | 2 + .../management/routing-analytics-routes.ts | 37 ++ tests/routing-analytics.test.ts | 201 ++++++++++ 6 files changed, 644 insertions(+), 1 deletion(-) create mode 100644 src/routing/analytics.ts create mode 100644 src/server/management/routing-analytics-routes.ts create mode 100644 tests/routing-analytics.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 5d9a9d75d0..22da9f394f 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -41,7 +41,7 @@ other; closing one is a maintainer decision and neither is stale. |---|---|---|---|---|---|---| | RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | DRAFT OPEN | | RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress | -| RI-03 | `feat/ri-03-routing-analytics` | `feat/ri-02` head | pending | pending | pending | queued | +| RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress | | RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | | RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | | RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | @@ -112,3 +112,24 @@ other; closing one is a maintainer decision and neither is stale. codex-routing, codex-account-namespaces) - `bun run privacy:scan`: passed - Remaining Low findings: none + +### RI-03 - feat/ri-03-routing-analytics + +- Base SHA: `7efb6e84284c070c155c2e5254f1400917df31a1` (RI-02 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 3 fixed pre-push - (1) `requestHistoryDb` accessor + missing from the indexer (analytics needs the handle after open); + (2) SQL column names are snake_case - analytics SELECT now aliases to + camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not + `costUsd`; plus the row-cap is injectable for truncation tests. +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/routing-analytics.test.ts`: 8/8 pass (32 assertions): + classification (success/failure/cancel/incomplete), percentiles + + coverage, fallback rate, provider/model/account + profile breakdown, + unknown-price honesty, filters, truncation flag, API payload + - Focused regression suites: 144/144 pass across 6 files + - `bun run privacy:scan`: passed +- Remaining Low findings: none diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts new file mode 100644 index 0000000000..7ddfed9ef8 --- /dev/null +++ b/src/routing/analytics.ts @@ -0,0 +1,372 @@ +/** + * Source-backed routing analytics (RI-03). + * + * All metrics derive from the rebuildable request-history index + * (`routing-history.sqlite`), never from repeated full JSONL scans. The + * analysis is read-only: no routing decision, profile, or weight changes here + * (ADR-10 - no automatic self-tuning). + * + * Bounds: at most ANALYTICS_MAX_ROWS matching rows are analyzed per call; a + * larger population sets `historyTruncated: true` so readers never mistake a + * sample for the full history. + */ + +import type { PersistedUsageEntry, PersistedUsageAttempt } from "../usage/log"; +import { estimateRequestCost, serviceTierContext } from "../usage/cost"; +import { openRequestHistoryIndex, requestHistoryDb } from "./history/indexer"; + +export const ANALYTICS_MAX_ROWS = 50_000; + +export interface RoutingAnalyticsFilters { + provider?: string; + model?: string; + profileId?: string; + surface?: string; + from?: number; + to?: number; +} + +export type AnalyticsConfidence = "high" | "medium" | "low"; + +export interface AnalyticsBreakdownRow { + provider: string; + model: string; + accountRef?: string; + profileId?: string; + requests: number; + successes: number; + failures: number; + cancelled: number; + successRate: number | null; + p50DurationMs?: number; + estimatedCostUsdPerSuccessfulRequest?: number | null; +} + +export interface AnalyticsProfileRow { + profileId: string; + profileRevision?: string; + requests: number; + successes: number; + failures: number; + fallbacks: number; + successRate: number | null; +} + +export interface RoutingAnalyticsResult { + generatedAt: number; + totalRequests: number; + scannedRows: number; + historyTruncated: boolean; + confidence: AnalyticsConfidence | null; + successRate: number | null; + failureRate: number | null; + cancelledRate: number | null; + fallbackRate: number | null; + totalAttempts: number; + averageAttemptsPerRequest: number | null; + incompleteStreamRate: number | null; + cooldownTriggeringFailures: number; + durationMs: { + p50?: number; + p95?: number; + p99?: number; + sampleCount: number; + }; + firstOutputMs: { + p50?: number; + p95?: number; + p99?: number; + sampleCount: number; + /** Share of scanned requests with a TTFT measurement (0..1). */ + coverage: number | null; + }; + estimatedCostUsdPerSuccessfulRequest: number | null; + estimatedCostUsdTotalSuccessful: number | null; + usageCoverage: number | null; + priceCoverage: number | null; + breakdown: AnalyticsBreakdownRow[]; + profileBreakdown: AnalyticsProfileRow[]; +} + +interface ScannedRow { + provider: string; + model: string; + apiKeyId?: string | null; + profileId?: string | null; + profileRevision?: string | null; + status: number; + durationMs: number; + firstOutputMs?: number | null; + closeReason?: string | null; + terminalStatus?: string | null; + usageStatus: string; + usageJson?: string | null; + attemptCount: number; + fallback: number; + rowJson: string; +} + +interface Bucket extends AnalyticsBreakdownRow { + durations: number[]; + costUsdSum: number; + costRows: number; +} + +const COOLDOWN_RECOVERY_KINDS = new Set([ + "rate-limit-429", + "key-429", + "oauth-401", + "anthropic-oauth-429", +]); + +function percentile(sorted: number[], p: number): number | undefined { + if (sorted.length === 0) return undefined; + const index = Math.max(0, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.min(index, sorted.length - 1)]; +} + +function classifyRow(row: ScannedRow): "success" | "failure" | "cancelled" { + if (row.closeReason === "client_cancel" || row.status === 499) return "cancelled"; + if (row.terminalStatus === "incomplete") return "failure"; + if (row.terminalStatus && row.terminalStatus !== "completed") return "failure"; + if (row.status >= 400) return "failure"; + return "success"; +} + +function parseEntry(rowJson: string): PersistedUsageEntry | null { + try { + const parsed = JSON.parse(rowJson) as PersistedUsageEntry; + return parsed && typeof parsed === "object" && typeof parsed.requestId === "string" ? parsed : null; + } catch { + return null; + } +} + +function attemptsOf(entry: PersistedUsageEntry | null): PersistedUsageAttempt[] | undefined { + return entry?.attempts; +} + +function cooldownTriggering(entry: PersistedUsageEntry | null, status: number): boolean { + if (status === 429) return true; + const attempts = attemptsOf(entry) ?? []; + return attempts.some(attempt => attempt.recoveryKinds.some(kind => COOLDOWN_RECOVERY_KINDS.has(kind))); +} + +function isSuccessStatus(status: number): boolean { + return status >= 200 && status < 400; +} + +export async function computeRoutingAnalytics( + filters: RoutingAnalyticsFilters, + options: { maxRows?: number } = {}, +): Promise { + await openRequestHistoryIndex(); + const handle = requestHistoryDb(); + const maxRows = Math.min( + Math.max(1, Math.trunc(options.maxRows ?? ANALYTICS_MAX_ROWS)), + ANALYTICS_MAX_ROWS, + ); + + const where: string[] = []; + const values: Array = []; + const add = (clause: string, value: string | number) => { + where.push(clause); + values.push(value); + }; + if (filters.provider !== undefined) add("provider = ?", filters.provider); + if (filters.model !== undefined) add("model = ?", filters.model); + if (filters.profileId !== undefined) add("profile_id = ?", filters.profileId); + if (filters.surface !== undefined) add("surface = ?", filters.surface); + if (filters.from !== undefined) add("timestamp >= ?", filters.from); + if (filters.to !== undefined) add("timestamp <= ?", filters.to); + const whereSql = where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""; + + const rows = handle.query( + `SELECT provider, model, api_key_id AS apiKeyId, profile_id AS profileId, + profile_revision AS profileRevision, status, + duration_ms AS durationMs, first_output_ms AS firstOutputMs, + close_reason AS closeReason, terminal_status AS terminalStatus, + usage_status AS usageStatus, usage_json AS usageJson, + attempt_count AS attemptCount, fallback, row_json AS rowJson + FROM requests${whereSql} ORDER BY timestamp DESC LIMIT ?`, + ).all(...values, maxRows + 1) as ScannedRow[]; + + const scanned = rows.slice(0, maxRows); + const historyTruncated = rows.length > maxRows; + + let successes = 0; + let failures = 0; + let cancelled = 0; + let fallbacks = 0; + let totalAttempts = 0; + let incompleteStreams = 0; + let cooldownFailures = 0; + let usageReported = 0; + const durations: number[] = []; + const firstOutputs: number[] = []; + let costTotalUsd = 0; + let costCount = 0; + + const byKey = new Map(); + const byProfile = new Map(); + + for (const row of scanned) { + const kind = classifyRow(row); + if (kind === "success") successes += 1; + else if (kind === "failure") failures += 1; + else cancelled += 1; + if (row.fallback === 1) fallbacks += 1; + totalAttempts += row.attemptCount; + if (row.terminalStatus === "incomplete") incompleteStreams += 1; + durations.push(row.durationMs); + if (row.firstOutputMs !== null && row.firstOutputMs !== undefined && row.firstOutputMs >= 0) { + firstOutputs.push(row.firstOutputMs); + } + if (row.usageStatus !== "unreported") usageReported += 1; + + const entry = kind === "success" || row.status >= 400 ? parseEntry(row.rowJson) : null; + if (cooldownTriggering(entry, row.status)) cooldownFailures += 1; + + if (kind === "success" && entry?.usage) { + const estimate = estimateRequestCost({ + provider: row.provider, + model: row.model, + usage: entry.usage, + usageStatus: entry.usageStatus, + serviceTier: serviceTierContext(entry), + }); + if (estimate) { + costTotalUsd += estimate.cost.total; + costCount += 1; + } + } + + const key = `${row.provider}\0${row.model}\0${row.apiKeyId ?? ""}\0${row.profileId ?? ""}`; + let bucket: Bucket | undefined = byKey.get(key); + if (!bucket) { + bucket = { + provider: row.provider, + model: row.model, + ...(row.apiKeyId ? { accountRef: row.apiKeyId } : {}), + ...(row.profileId ? { profileId: row.profileId } : {}), + requests: 0, + successes: 0, + failures: 0, + cancelled: 0, + successRate: null, + durations: [], + costUsdSum: 0, + costRows: 0, + }; + byKey.set(key, bucket); + } + bucket.requests += 1; + if (kind === "success") bucket.successes += 1; + else if (kind === "failure") bucket.failures += 1; + else bucket.cancelled += 1; + bucket.durations.push(row.durationMs); + if (kind === "success" && entry?.usage) { + const estimate = estimateRequestCost({ + provider: row.provider, + model: row.model, + usage: entry.usage, + usageStatus: entry.usageStatus, + serviceTier: serviceTierContext(entry), + }); + if (estimate) { + bucket.costUsdSum += estimate.cost.total; + bucket.costRows += 1; + } + } + + if (row.profileId) { + let profile = byProfile.get(row.profileId); + if (!profile) { + profile = { + profileId: row.profileId, + ...(row.profileRevision ? { profileRevision: row.profileRevision } : {}), + requests: 0, + successes: 0, + failures: 0, + fallbacks: 0, + successRate: null, + }; + byProfile.set(row.profileId, profile); + } + profile.requests += 1; + if (kind === "success") profile.successes += 1; + else if (kind === "failure") profile.failures += 1; + if (row.fallback === 1) profile.fallbacks += 1; + } + } + + durations.sort((a, b) => a - b); + firstOutputs.sort((a, b) => a - b); + const total = scanned.length; + const rate = (count: number): number | null => (total > 0 ? count / total : null); + + const breakdown: AnalyticsBreakdownRow[] = [...byKey.values()].map(bucket => { + const sorted = bucket.durations.sort((a, b) => a - b); + return { + provider: bucket.provider, + model: bucket.model, + ...(bucket.accountRef ? { accountRef: bucket.accountRef } : {}), + ...(bucket.profileId ? { profileId: bucket.profileId } : {}), + requests: bucket.requests, + successes: bucket.successes, + failures: bucket.failures, + cancelled: bucket.cancelled, + successRate: bucket.requests > 0 ? bucket.successes / bucket.requests : null, + ...(percentile(sorted, 50) !== undefined ? { p50DurationMs: percentile(sorted, 50) } : {}), + ...(bucket.requests > 0 + ? { estimatedCostUsdPerSuccessfulRequest: bucket.costRows > 0 + ? bucket.costUsdSum / bucket.costRows + : null } + : {}), + }; + }).sort((a, b) => b.requests - a.requests); + + const profileBreakdown: AnalyticsProfileRow[] = [...byProfile.values()].map(profile => ({ + ...profile, + successRate: profile.requests > 0 ? profile.successes / profile.requests : null, + })).sort((a, b) => b.requests - a.requests); + + const confidence: AnalyticsConfidence | null = total === 0 + ? null + : total >= 100 ? "high" : total >= 20 ? "medium" : "low"; + + return { + generatedAt: Date.now(), + totalRequests: total, + scannedRows: scanned.length, + historyTruncated, + confidence, + successRate: rate(successes), + failureRate: rate(failures), + cancelledRate: rate(cancelled), + fallbackRate: rate(fallbacks), + totalAttempts, + averageAttemptsPerRequest: total > 0 ? totalAttempts / total : null, + incompleteStreamRate: rate(incompleteStreams), + cooldownTriggeringFailures: cooldownFailures, + durationMs: { + ...(percentile(durations, 50) !== undefined ? { p50: percentile(durations, 50) } : {}), + ...(percentile(durations, 95) !== undefined ? { p95: percentile(durations, 95) } : {}), + ...(percentile(durations, 99) !== undefined ? { p99: percentile(durations, 99) } : {}), + sampleCount: durations.length, + }, + firstOutputMs: { + ...(percentile(firstOutputs, 50) !== undefined ? { p50: percentile(firstOutputs, 50) } : {}), + ...(percentile(firstOutputs, 95) !== undefined ? { p95: percentile(firstOutputs, 95) } : {}), + ...(percentile(firstOutputs, 99) !== undefined ? { p99: percentile(firstOutputs, 99) } : {}), + sampleCount: firstOutputs.length, + coverage: total > 0 ? firstOutputs.length / total : null, + }, + estimatedCostUsdPerSuccessfulRequest: costCount > 0 ? costTotalUsd / costCount : null, + estimatedCostUsdTotalSuccessful: costCount > 0 ? costTotalUsd : null, + usageCoverage: total > 0 ? usageReported / total : null, + priceCoverage: successes > 0 ? costCount / successes : null, + breakdown, + profileBreakdown, + }; +} diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index 4e4720bde6..287c876b5c 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -541,3 +541,13 @@ export async function requestHistoryRowById(requestId: string): Promise { return openRequestHistoryIndex(); } + +/** + * Raw handle for analytics-style queries. Callers must await + * `openRequestHistoryIndex()` first; the handle is valid until + * `closeRequestHistoryIndex()`. + */ +export function requestHistoryDb(): Database { + if (!db) throw new Error("request-history index is not open"); + return db; +} diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 63129d5dfe..af8fd499db 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -60,6 +60,7 @@ import type { ManagementApiDeps } from "./management/context"; import { handleConfigRoutes } from "./management/config-routes"; import { handleLogsUsageRoutes } from "./management/logs-usage-routes"; import { handleRequestHistoryRoutes } from "./management/request-history-routes"; +import { handleRoutingAnalyticsRoutes } from "./management/routing-analytics-routes"; import { handleProviderRoutes } from "./management/provider-routes"; import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; @@ -140,6 +141,7 @@ export async function handleManagementAPI( routed = (await handleConfigRoutes(ctx)) ?? (await handleLogsUsageRoutes(ctx)) ?? (await handleRequestHistoryRoutes(ctx)) + ?? (await handleRoutingAnalyticsRoutes(ctx)) ?? (await handleProviderRoutes(ctx)) ?? (await handleModelRoutes(ctx)) ?? (await handleIntegrationRoutes(ctx)) diff --git a/src/server/management/routing-analytics-routes.ts b/src/server/management/routing-analytics-routes.ts new file mode 100644 index 0000000000..2e06199eb3 --- /dev/null +++ b/src/server/management/routing-analytics-routes.ts @@ -0,0 +1,37 @@ +/** + * Routing analytics API (RI-03): `GET /api/routing-analytics`. + * + * Returns source-backed reliability/latency/cost metrics over the + * request-history index. Read-only; never changes routing behavior. + */ + +import { computeRoutingAnalytics } from "../../routing/analytics"; +import { jsonResponse } from "../auth-cors"; +import type { ManagementContext } from "./context"; + +function parseOptionalInt(raw: string | null): number | undefined { + if (raw === null) return undefined; + const value = Number(raw.trim()); + return Number.isInteger(value) ? value : undefined; +} + +export async function handleRoutingAnalyticsRoutes(ctx: ManagementContext): Promise { + const { url, req, config } = ctx; + if (url.pathname !== "/api/routing-analytics" || req.method !== "GET") return null; + + const from = parseOptionalInt(url.searchParams.get("from")); + const to = parseOptionalInt(url.searchParams.get("to")); + if (from !== undefined && to !== undefined && from > to) { + return jsonResponse({ error: { code: "invalid_range", message: "from must not be after to" } }, 400, req, config); + } + + const result = await computeRoutingAnalytics({ + provider: url.searchParams.get("provider")?.trim() || undefined, + model: url.searchParams.get("model")?.trim() || undefined, + profileId: url.searchParams.get("profileId")?.trim() || undefined, + surface: url.searchParams.get("surface")?.trim() || undefined, + from, + to, + }); + return jsonResponse(result, 200, req, config); +} diff --git a/tests/routing-analytics.test.ts b/tests/routing-analytics.test.ts new file mode 100644 index 0000000000..7c3657b044 --- /dev/null +++ b/tests/routing-analytics.test.ts @@ -0,0 +1,201 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + appendUsageEntry, + resetUsageReadCacheForTests, + type PersistedUsageEntry, +} from "../src/usage/log"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; +import { computeRoutingAnalytics } from "../src/routing/analytics"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +function entry( + requestId: string, + overrides: Partial & { timestamp: number; status: number; durationMs: number }, +): PersistedUsageEntry { + return { + requestId, + provider: "a", + model: "m1", + usageStatus: "reported", + ...overrides, + }; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-analytics-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"] } }, + }; +} + +describe("routing analytics (RI-03)", () => { + test("classifies success, failure, cancellation and incomplete streams", async () => { + appendUsageEntry(entry("r1", { timestamp: 1000, status: 200, durationMs: 100, firstOutputMs: 10 })); + appendUsageEntry(entry("r2", { timestamp: 2000, status: 200, durationMs: 200, firstOutputMs: 30 })); + appendUsageEntry(entry("r3", { timestamp: 3000, status: 429, durationMs: 300 })); + appendUsageEntry(entry("r4", { timestamp: 4000, status: 499, durationMs: 50, closeReason: "client_cancel" })); + appendUsageEntry(entry("r5", { timestamp: 5000, status: 200, durationMs: 400, terminalStatus: "incomplete" })); + + const result = await computeRoutingAnalytics({}); + expect(result.totalRequests).toBe(5); + expect(result.successRate).toBe(0.4); + expect(result.failureRate).toBe(0.4); + expect(result.cancelledRate).toBe(0.2); + expect(result.incompleteStreamRate).toBe(0.2); + expect(result.cooldownTriggeringFailures).toBe(1); + expect(result.confidence).toBe("low"); + expect(result.historyTruncated).toBe(false); + }); + + test("computes duration and TTFT percentiles with coverage", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, firstOutputMs: 10 })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 200, firstOutputMs: 20 })); + appendUsageEntry(entry("r3", { timestamp: 3, status: 200, durationMs: 300, firstOutputMs: 30 })); + appendUsageEntry(entry("r4", { timestamp: 4, status: 200, durationMs: 400 })); + + const result = await computeRoutingAnalytics({}); + // Nearest-rank percentiles over [100,200,300,400]: + expect(result.durationMs.p50).toBe(200); + expect(result.durationMs.p95).toBe(400); + expect(result.durationMs.p99).toBe(400); + expect(result.durationMs.sampleCount).toBe(4); + expect(result.firstOutputMs.p50).toBe(20); + expect(result.firstOutputMs.sampleCount).toBe(3); + expect(result.firstOutputMs.coverage).toBe(0.75); + }); + + test("fallback rate counts multi-attempt requests", async () => { + appendUsageEntry(entry("r1", { + timestamp: 1, + status: 200, + durationMs: 100, + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 50, sendCount: 1, recoveryKinds: ["transient-5xx"], usageStatus: "unreported" }, + { ordinal: 2, provider: "a", model: "m1", adapter: "openai-chat", status: 200, durationMs: 50, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100 })); + + const result = await computeRoutingAnalytics({}); + expect(result.fallbackRate).toBe(0.5); + expect(result.totalAttempts).toBe(3); + expect(result.averageAttemptsPerRequest).toBe(1.5); + }); + + test("breakdown groups by provider/model/account and profile", async () => { + appendUsageEntry(entry("r1", { + timestamp: 1, + status: 200, + durationMs: 100, + apiKeyId: "key-a", + routeDecision: { + version: 1, + decisionId: "d1", + createdAt: 1, + requestedModel: "policy/fast", + routeKind: "policy", + profile: { id: "fast", revision: "abc123" }, + requirements: [], + candidates: [{ provider: "a", model: "m1", eligible: true, exclusions: [] }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "policy" }, + }, + })); + appendUsageEntry(entry("r2", { + timestamp: 2, + status: 500, + durationMs: 200, + apiKeyId: "key-a", + routeDecision: { + version: 1, + decisionId: "d2", + createdAt: 2, + requestedModel: "policy/fast", + routeKind: "policy", + profile: { id: "fast", revision: "abc123" }, + requirements: [], + candidates: [{ provider: "a", model: "m1", eligible: true, exclusions: [] }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "policy" }, + }, + })); + + const result = await computeRoutingAnalytics({}); + expect(result.breakdown.length).toBe(1); + expect(result.breakdown[0]).toMatchObject({ + provider: "a", + model: "m1", + accountRef: "key-a", + profileId: "fast", + requests: 2, + successes: 1, + failures: 1, + successRate: 0.5, + }); + expect(result.profileBreakdown).toEqual([ + { profileId: "fast", profileRevision: "abc123", requests: 2, successes: 1, failures: 1, fallbacks: 0, successRate: 0.5 }, + ]); + }); + + test("usage and price coverage are honest about unknown data", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, usageStatus: "reported", usage: { inputTokens: 1000, outputTokens: 100 } })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100, usageStatus: "unreported" })); + + const result = await computeRoutingAnalytics({}); + expect(result.usageCoverage).toBe(0.5); + // Unknown price for provider "a": the estimate stays null, never zero. + expect(result.estimatedCostUsdPerSuccessfulRequest).toBeNull(); + expect(result.priceCoverage).toBe(0); + }); + + test("filters scope the analysis", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, provider: "a" })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100, provider: "b", model: "m2" })); + + const result = await computeRoutingAnalytics({ provider: "b" }); + expect(result.totalRequests).toBe(1); + expect(result.breakdown[0]).toMatchObject({ provider: "b", model: "m2" }); + }); + + test("explicit truncated-history indicator when the cap is hit", async () => { + for (let index = 0; index < 12; index++) { + appendUsageEntry(entry(`r${index}`, { timestamp: index, status: 200, durationMs: 10 })); + } + const result = await computeRoutingAnalytics({}, { maxRows: 10 }); + expect(result.scannedRows).toBe(10); + expect(result.historyTruncated).toBe(true); + }); + + test("API endpoint returns the analytics payload", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100 })); + const req = new ManagementRequest("http://localhost/api/routing-analytics", { method: "GET" }); + const response = await handleManagementAPI(req, new URL(req.url), config(), { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { totalRequests?: number; successRate?: number | null }; + expect(body.totalRequests).toBe(1); + expect(body.successRate).toBe(1); + }); +}); From 00e1c4ae5df32cdf6d72957c1a36b334fe4dc0a6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:43:30 +0200 Subject: [PATCH 03/10] feat(routing): add validated routing policy profiles (RI-04) --- .../001_pr_stack_status.md | 27 +- .../docs/reference/configuration/routing.md | 64 +++ src/cli/index.ts | 13 +- src/cli/route-policy.ts | 89 ++++ src/config.ts | 22 + src/routing/evaluator.ts | 249 +++++++++++ src/routing/profile.ts | 403 ++++++++++++++++++ src/routing/trace.ts | 11 +- src/server/management-api.ts | 2 + .../management/routing-profile-routes.ts | 111 +++++ src/types.ts | 64 +++ tests/routing-profile.test.ts | 287 +++++++++++++ 12 files changed, 1334 insertions(+), 8 deletions(-) create mode 100644 src/cli/route-policy.ts create mode 100644 src/routing/evaluator.ts create mode 100644 src/routing/profile.ts create mode 100644 src/server/management/routing-profile-routes.ts create mode 100644 tests/routing-profile.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 22da9f394f..7be9dc428c 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -42,7 +42,7 @@ other; closing one is a maintainer decision and neither is stale. | RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | DRAFT OPEN | | RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress | | RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress | -| RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | +| RI-04 | `feat/ri-04-policy-profile-core` | `2069e724e` (RI-03 head) | pending | pending | pending | in progress | | RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | | RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | @@ -133,3 +133,28 @@ other; closing one is a maintainer decision and neither is stale. - Focused regression suites: 144/144 pass across 6 files - `bun run privacy:scan`: passed - Remaining Low findings: none + +### RI-04 - feat/ri-04-policy-profile-core + +- Base SHA: `2069e724ec27e176644a11ff55bef307f5ebe3bf` (RI-03 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 4 fixed pre-push - (1) `serviceTier` evidence type + was `Unknownable` (number|boolean) but service tiers are strings - trace + type narrowed to `string | "unknown"`; (2) alias validation missed the + reserved `combo/` namespace prefix; (3) trace candidates did not carry + `score` - added `score` to `TraceCandidateInput`/`buildCandidate`; + (4) test expectation for weight normalization used wrong math (unspecified + weights keep defaults; sum 4.35 not 4). +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/routing-profile.test.ts`: 12/12 pass (validation, + normalization, revision digest, collisions, config load, id/alias + resolution, dry-run eligibility/unknown/tie-break, API list+dry-run, + API error codes) + - Focused regression suites: 176/176 pass across 8 files + - `bun run privacy:scan`: passed + - `tests/config.test.ts`: 109/115 pass; the 6 symlink failures reproduce + identically on the pristine base (Windows symlink EPERM, environmental) +- Remaining Low findings: none diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 85a5c89176..6e56be384c 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -11,6 +11,7 @@ Routing turns the model id sent by a client into one concrete provider and upstr | --- | --- | --- | --- | | `defaultProvider` | `string` | `"openai"` | Final provider used when no earlier model rule matches. It must name an enabled configured provider. | | `combos?` | `Record` | `{}` | Virtual `combo/` models built from ordered provider/model targets. | +| `routingProfiles?` | `Record` | `{}` | Virtual `policy/` models that select among an explicit candidate allowlist using hard capability requirements and deterministic scoring. | ## Model resolution order @@ -82,6 +83,69 @@ namespace, and cannot use reserved bare native families such as `gpt-*`, `o1-*`, For strategy behavior, retryable failures, cooldowns, encrypted v2 task limits, and management commands, see [Combos](/guides/combos/). +## Routing policy profiles (`config.routingProfiles`) + +Routing policy profiles are the Router Intelligence selection layer: an explicitly requested +`policy/` (or configured alias) chooses among a fixed candidate allowlist using hard capability +requirements and deterministic, explainable scoring. Existing model ids are **never** routed through +a profile implicitly - policy routing only activates when the client requests a policy model id. + +Each key is an id matching `[A-Za-z0-9][A-Za-z0-9._-]{0,63}`, always addressable as `policy/`, +with one optional `alias`. Aliases must be unique and cannot collide with configured providers, +combos, codex account namespaces, the `policy/` namespace, or reserved bare native families +(`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `codex-*`). + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `candidates` | `{ provider: string; model: string }[]` | required | Explicit allowlist of `provider/model` refs. No implicit expansion. | +| `alias?` | `string` | — | Optional public model id in place of `policy/`. | +| `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | +| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights; normalized deterministically. | +| `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd`. | +| `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | + +`require` supports: `minContextWindow` (positive integer), and the booleans `tools`, `imageInput`, +`structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`; plus `reasoningEffort` and +`serviceTier` strings. + +```json +{ + "routingProfiles": { + "fast": { + "alias": "ocx/fast", + "candidates": [ + { "provider": "anthropic", "model": "claude-sonnet-5" }, + { "provider": "openai", "model": "gpt-5.6-sol" } + ], + "require": { "tools": true, "minContextWindow": 128000 }, + "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, + "limits": { "maxEstimatedCostUsd": 0.50 }, + "unknownEvidence": { + "capability": "exclude", + "health": "penalize", + "quota": "penalize", + "cost": "penalize" + } + } + } +} +``` + +CLI: `ocx route policy list`, `ocx route policy show `, and +`ocx route policy dry-run --model-context --tools`. Dry-run evaluates candidates +without sending any upstream request. + +### Combos vs policy profiles + +- A **combo** is explicit ordered/weighted target routing and failover: the configured order (or + smooth weighted round-robin) decides, and failures advance through the list. +- A **policy profile** is evidence-based selection among configured candidates: hard capability + requirements filter first, then deterministic scoring ranks the survivors. + +Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate +is chosen. Profile scoring expands in capability (RI-05), health (RI-06), quota (RI-07), and cost +(RI-08) dimensions, each recorded in the per-request route-decision trace. + ### Catalog eligibility A combo remains directly routable even when it cannot be listed. `ocx sync`, `/v1/models`, and the diff --git a/src/cli/index.ts b/src/cli/index.ts index 8557613596..afaa440934 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1007,13 +1007,18 @@ switch (command) { break; } case "route": { - if (args[1] !== "combo") { - console.error("Usage: ocx route combo "); + if (args[1] !== "combo" && args[1] !== "policy") { + console.error("Usage: ocx route "); process.exitCode = 2; break; } - const { handleComboCommand } = await import("./combo"); - process.exitCode = await handleComboCommand(args.slice(2)); + if (args[1] === "combo") { + const { handleComboCommand } = await import("./combo"); + process.exitCode = await handleComboCommand(args.slice(2)); + } else { + const { handleRoutePolicyCommand } = await import("./route-policy"); + process.exitCode = await handleRoutePolicyCommand(args.slice(2)); + } break; } case "agent": { diff --git a/src/cli/route-policy.ts b/src/cli/route-policy.ts new file mode 100644 index 0000000000..7753e74358 --- /dev/null +++ b/src/cli/route-policy.ts @@ -0,0 +1,89 @@ +import { + CliUsageError, + printData, + rejectArgs, + runCliAction, + runtimeRequest, + takeFlag, + takeIntegerOption, + type RuntimeApiDeps, +} from "./runtime-api"; + +const USAGE = `Usage: + ocx route policy list [--json] + ocx route policy show [--json] + ocx route policy dry-run [--model-context ] [--tools] + [--image] [--structured-output] [--json]`; + +interface ProfileRow { + id?: string; + model?: string; + revision?: string; +} + +async function list(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + rejectArgs(args, USAGE); + const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps); + const rows = result.profiles ?? []; + printData( + result, + wantsJson, + rows.length + ? rows.map(row => `${String(row.id)} ${String(row.model ?? `policy/${row.id}`)} rev:${String(row.revision ?? "-")}`) + : ["No routing profiles configured."], + ); +} + +async function show(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift(); + const wantsJson = takeFlag(args, "--json"); + if (!id) throw new CliUsageError("profile id is required", USAGE); + rejectArgs(args, USAGE); + const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps); + const profile = (result.profiles ?? []).find(candidate => candidate.id === id); + if (!profile) throw new CliUsageError(`unknown routing profile: ${id}`, USAGE); + printData(profile, wantsJson, wantsJson ? undefined : [JSON.stringify(profile, null, 2)]); +} + +async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift(); + const wantsJson = takeFlag(args, "--json"); + if (!id) throw new CliUsageError("profile id is required", USAGE); + const modelContext = takeIntegerOption(args, "--model-context", { min: 1 }); + const tools = takeFlag(args, "--tools"); + const image = takeFlag(args, "--image"); + const structuredOutput = takeFlag(args, "--structured-output"); + rejectArgs(args, USAGE); + const result = await runtimeRequest( + "/api/routing-profiles/dry-run", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: id, + evidence: { + ...(modelContext !== undefined ? { contextWindow: modelContext } : {}), + ...(tools ? { toolsRequired: true } : {}), + ...(image ? { imageInputRequired: true } : {}), + ...(structuredOutput ? { structuredOutputRequired: true } : {}), + }, + }), + }, + deps, + ); + printData(result, wantsJson, wantsJson ? undefined : [JSON.stringify(result, null, 2)]); +} + +export async function handleRoutePolicyCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise { + return runCliAction(async () => { + const [sub, ...rest] = argv; + if (sub === "list") await list(rest, deps); + else if (sub === "show") await show(rest, deps); + else if (sub === "dry-run") await dryRun(rest, deps); + else throw new CliUsageError(`unknown route policy command: ${sub ?? ""}`, USAGE); + }); +} diff --git a/src/config.ts b/src/config.ts index 72defeb9ca..b9b9d7c09f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,7 @@ import { MAIN_CODEX_ACCOUNT_NAMESPACE_TARGET, } from "./codex/account-namespace-match"; import { COMBO_NAMESPACE, comboConfigIssues } from "./combos/types"; +import { routingProfileIssues } from "./routing/profile"; import { forgetEphemeralSecretPath, hardenSecretDir, @@ -1261,6 +1262,27 @@ const configSchema = z.object({ } } } + const routingProfiles = (config as { routingProfiles?: unknown }).routingProfiles; + if (routingProfiles !== undefined) { + if (!routingProfiles || typeof routingProfiles !== "object" || Array.isArray(routingProfiles)) { + ctx.addIssue({ code: "custom", path: ["routingProfiles"], message: "routingProfiles must be an object" }); + } else { + for (const [id, raw] of Object.entries(routingProfiles as Record)) { + for (const issue of routingProfileIssues(id, raw, { + providers: config.providers, + combos: combos as Record | undefined, + routingProfiles: routingProfiles as Record, + codexAccountNamespaces: accountNamespaces, + }, { excludeProfileId: id })) { + ctx.addIssue({ + code: "custom", + path: ["routingProfiles", id, ...issue.path], + message: issue.message, + }); + } + } + } + } }); /** diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts new file mode 100644 index 0000000000..3e550dc178 --- /dev/null +++ b/src/routing/evaluator.ts @@ -0,0 +1,249 @@ +/** + * Deterministic policy-profile evaluator (RI-04 core; extended by RI-05..08). + * + * RI-04 evaluates hard capability requirements against supplied evidence and + * scores by deterministic configured priority only. It never dispatches an + * upstream request; execution wiring arrives with RI-05. + */ + +import type { OcxConfig } from "../types"; +import { + buildRouteDecisionTrace, + type RouteCapabilityEvidence, + type RouteCostEvidence, + type RouteDecisionTraceV1, + type RouteExclusionReason, + type RouteHealthEvidence, + type RouteQuotaEvidence, + type RouteRequirementEvidence, + type RouteScoreEvidence, + type Unknownable, +} from "./trace"; +import { getRoutingProfile, type NormalizedRoutingProfile } from "./profile"; + +export interface PolicyRequestEvidence { + /** Required context window for this request (tokens). */ + contextWindow?: number; + toolsRequired?: boolean; + imageInputRequired?: boolean; + structuredOutputRequired?: boolean; + reasoningEffort?: string; + serviceTier?: string; + encryptedCodexTask?: boolean; +} + +export interface PolicyCandidateEvidence { + provider: string; + model: string; + accountRef?: string; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; +} + +export interface PolicyEvaluationCandidate { + provider: string; + model: string; + accountRef?: string; + eligible: boolean; + exclusions: RouteExclusionReason[]; + requirements: RouteRequirementEvidence[]; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; + score?: RouteScoreEvidence; +} + +export interface PolicyEvaluationResult { + profileId: string; + profileRevision: string; + candidates: PolicyEvaluationCandidate[]; + selectedIndex: number | null; + trace: RouteDecisionTraceV1; +} + +function booleanRequirement( + id: string, + required: boolean | undefined, + actual: Unknownable | undefined, +): RouteRequirementEvidence | null { + if (required === undefined) return null; + if (actual === undefined || actual === "unknown") { + return { id, expected: required, outcome: "unknown" }; + } + return { + id, + expected: required, + actual: typeof actual === "boolean" ? actual : String(actual), + outcome: typeof actual === "boolean" && actual === required ? "satisfied" : "unsatisfied", + }; +} + +function requirementFor( + require: NormalizedRoutingProfile["require"], + capability: RouteCapabilityEvidence | undefined, +): RouteRequirementEvidence[] { + const requirements: RouteRequirementEvidence[] = []; + if (require.minContextWindow !== undefined) { + const actual = capability?.contextWindow; + if (typeof actual === "number") { + requirements.push({ + id: "min-context-window", + expected: require.minContextWindow, + actual, + outcome: actual >= require.minContextWindow ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "min-context-window", expected: require.minContextWindow, outcome: "unknown" }); + } + } + const tools = booleanRequirement("tools", require.tools, capability?.tools); + if (tools) requirements.push(tools); + const image = booleanRequirement("image-input", require.imageInput, capability?.image); + if (image) requirements.push(image); + const structured = booleanRequirement("structured-output", require.structuredOutput, capability?.structuredOutput); + if (structured) requirements.push(structured); + if (require.reasoningEffort !== undefined) { + const ladder = capability?.reasoningEfforts; + if (Array.isArray(ladder)) { + requirements.push({ + id: "reasoning-effort", + expected: require.reasoningEffort, + actual: ladder.join(","), + outcome: ladder.includes(require.reasoningEffort) ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "reasoning-effort", expected: require.reasoningEffort, outcome: "unknown" }); + } + } + if (require.serviceTier !== undefined) { + const actual = capability?.serviceTier; + if (actual === undefined || actual === "unknown") { + requirements.push({ id: "service-tier", expected: require.serviceTier, outcome: "unknown" }); + } else { + requirements.push({ + id: "service-tier", + expected: require.serviceTier, + actual: String(actual), + outcome: actual === require.serviceTier ? "satisfied" : "unsatisfied", + }); + } + } + const local = booleanRequirement("local-only", require.localOnly, capability?.localOnly); + if (local) requirements.push(local); + const remote = booleanRequirement("remote-allowed", require.remoteAllowed, capability?.remoteAllowed); + if (remote) requirements.push(remote); + const encrypted = booleanRequirement( + "encrypted-codex-tasks", + require.encryptedCodexTasks, + capability?.encryptedCodexTasks, + ); + if (encrypted) requirements.push(encrypted); + return requirements; +} + +function unsatisfiedOrUnknown(requirements: RouteRequirementEvidence[]): RouteRequirementEvidence[] { + return requirements.filter(requirement => requirement.outcome !== "satisfied"); +} + +function configuredPriorityScore(index: number, total: number): number { + return total > 1 ? (total - index) / total : 1; +} + +/** + * Evaluate a profile against request + candidate evidence. Deterministic: + * candidates are scored in declaration order, ties break by earlier index. + */ +export function evaluatePolicyProfile( + config: OcxConfig, + profileId: string, + requestEvidence: PolicyRequestEvidence, + candidateEvidence: PolicyCandidateEvidence[], +): PolicyEvaluationResult { + const profile = getRoutingProfile(config, profileId); + if (!profile) throw new Error(`Unknown routing profile: ${profileId}`); + + const candidates: PolicyEvaluationCandidate[] = []; + let selectedIndex: number | null = null; + let bestScore = Number.NEGATIVE_INFINITY; + + profile.candidates.forEach((declared, index) => { + const evidence = candidateEvidence.find( + candidate => candidate.provider === declared.provider && candidate.model === declared.model, + ) ?? { provider: declared.provider, model: declared.model }; + const requirements = requirementFor(profile.require, evidence.capability); + const exclusions: RouteExclusionReason[] = []; + const bad = unsatisfiedOrUnknown(requirements); + for (const requirement of bad) { + if (requirement.outcome === "unsatisfied") { + exclusions.push({ code: "capability-unsatisfied", detail: requirement.id }); + } else { + exclusions.push({ code: "unknown-capability", detail: requirement.id }); + } + } + const unsatisfied = bad.some(requirement => requirement.outcome === "unsatisfied"); + const unknown = bad.some(requirement => requirement.outcome === "unknown"); + // Unknown capability handling per profile: exclude (default), penalize, + // or allow. "penalize" currently cannot move the score because RI-04 has + // no capability component yet - the capability score arrives with RI-05. + const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; + const eligible = !unsatisfied && !excludedByUnknown; + + const score: RouteScoreEvidence = { + total: configuredPriorityScore(index, profile.candidates.length), + components: { configuredPriority: configuredPriorityScore(index, profile.candidates.length) }, + }; + const evaluated: PolicyEvaluationCandidate = { + provider: evidence.provider, + model: evidence.model, + ...(evidence.accountRef ? { accountRef: evidence.accountRef } : {}), + eligible, + exclusions, + requirements, + ...(evidence.capability ? { capability: evidence.capability } : {}), + ...(evidence.health ? { health: evidence.health } : {}), + ...(evidence.quota ? { quota: evidence.quota } : {}), + ...(evidence.cost ? { cost: evidence.cost } : {}), + score, + }; + candidates.push(evaluated); + + if (evaluated.eligible && score.total > bestScore) { + bestScore = score.total; + selectedIndex = index; + } + }); + + const trace = buildRouteDecisionTrace({ + requestedModel: `policy/${profileId}`, + routeKind: "policy", + profile: { id: profile.id, revision: profile.revision }, + requirements: candidates.flatMap(candidate => candidate.requirements).slice(0, 16), + candidates: candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + ...(candidate.accountRef ? { accountRef: candidate.accountRef } : {}), + eligible: candidate.eligible, + exclusions: candidate.exclusions, + ...(candidate.score ? { score: candidate.score } : {}), + })), + selected: selectedIndex === null + ? { provider: candidates[0]?.provider ?? "", model: candidates[0]?.model ?? "", reason: "no-eligible-candidate" } + : { + candidateIndex: selectedIndex, + provider: candidates[selectedIndex]!.provider, + model: candidates[selectedIndex]!.model, + reason: "policy-selected", + }, + }); + + return { + profileId: profile.id, + profileRevision: profile.revision, + candidates, + selectedIndex, + trace, + }; +} diff --git a/src/routing/profile.ts b/src/routing/profile.ts new file mode 100644 index 0000000000..c0a724aa67 --- /dev/null +++ b/src/routing/profile.ts @@ -0,0 +1,403 @@ +/** + * Routing policy profiles (RI-04): schema validation, normalization, revision + * digest, and id/alias resolution. Mirrors the combos module discipline + * (`src/combos/types.ts`) so both virtual-routing namespaces stay consistent. + */ + +import { createHash } from "node:crypto"; +import type { + OcxConfig, + OcxRoutingProfileConfig, + OcxRoutingUnknownEvidenceMode, +} from "../types"; +import { codexAccountNamespaceEntries } from "../codex/account-namespaces"; +import { listComboIds, resolveComboId } from "../combos"; +import { hasOwnProvider } from "../config"; + +export const POLICY_NAMESPACE = "policy"; + +export const POLICY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +export const POLICY_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}(?:\/[A-Za-z0-9][A-Za-z0-9._-]{0,63})?$/; +export const NATIVE_OPENAI_FAMILY_PATTERN = /^(?:gpt-|o1-|o3-|o4-|codex-)/; + +export const DEFAULT_PROFILE_WEIGHTS = { + latency: 0.55, + health: 0.25, + cost: 0.10, + quota: 0.10, +} as const; + +export const DEFAULT_UNKNOWN_EVIDENCE: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode> = { + capability: "exclude", + health: "penalize", + quota: "penalize", + cost: "penalize", +}; + +export interface RoutingProfileValidationIssue { + path: Array; + message: string; +} + +export interface NormalizedRoutingProfileRequirements { + minContextWindow?: number; + tools?: boolean; + imageInput?: boolean; + structuredOutput?: boolean; + reasoningEffort?: string; + serviceTier?: string; + localOnly?: boolean; + remoteAllowed?: boolean; + encryptedCodexTasks?: boolean; +} + +export interface NormalizedRoutingProfile { + id: string; + alias: string | null; + candidates: Array<{ provider: string; model: string }>; + require: NormalizedRoutingProfileRequirements; + optimize: { latency: number; health: number; cost: number; quota: number }; + limits: { maxEstimatedCostUsd?: number }; + unknownEvidence: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode>; + revision: string; +} + +const REQUIRE_KEYS = [ + "minContextWindow", + "tools", + "imageInput", + "structuredOutput", + "reasoningEffort", + "serviceTier", + "localOnly", + "remoteAllowed", + "encryptedCodexTasks", +] as const; + +const UNKNOWN_EVIDENCE_KEYS = ["capability", "health", "quota", "cost"] as const; + +export function isValidPolicyId(id: string): boolean { + return POLICY_ID_PATTERN.test(id); +} + +export function policyModelId(id: string): string { + return `${POLICY_NAMESPACE}/${id}`; +} + +export function policyPublicModelId(id: string, profile: { alias?: string | null }): string { + const alias = typeof profile.alias === "string" ? profile.alias.trim() : ""; + return alias || policyModelId(id); +} + +export function parsePolicyModelId(modelId: string): string | null { + const slash = modelId.indexOf("/"); + if (slash <= 0 || modelId.slice(0, slash) !== POLICY_NAMESPACE) return null; + const id = modelId.slice(slash + 1); + return id.length > 0 ? id : null; +} + +/** + * Resolve a client-requested model id to a policy profile id. The canonical + * `policy/` form wins first; otherwise an exact alias match. + */ +export function resolvePolicyProfileId( + config: { routingProfiles?: Record }, + modelId: string, +): string | null { + const direct = parsePolicyModelId(modelId); + if (direct) return direct; + const profiles = config.routingProfiles; + if (!profiles) return null; + for (const [id, raw] of Object.entries(profiles)) { + if (!raw || typeof raw !== "object") continue; + const alias = typeof raw.alias === "string" ? raw.alias.trim() : ""; + if (alias && alias === modelId) return id; + } + return null; +} + +function aliasIssues( + id: string, + alias: string, + config: Pick, + options: { excludeProfileId?: string } = {}, +): RoutingProfileValidationIssue[] { + const issues: RoutingProfileValidationIssue[] = []; + if (!POLICY_ALIAS_PATTERN.test(alias)) { + issues.push({ + path: ["alias"], + message: "alias must use letters, numbers, dot, underscore, or hyphen, with at most one \"/\" segment", + }); + return issues; + } + if (alias === POLICY_NAMESPACE || alias.startsWith(`${POLICY_NAMESPACE}/`)) { + issues.push({ + path: ["alias"], + message: `alias must not use the reserved "${POLICY_NAMESPACE}/" namespace`, + }); + } + if (alias === "combo" || alias.startsWith("combo/")) { + issues.push({ + path: ["alias"], + message: `alias must not use the reserved "combo/" namespace`, + }); + } + if (!alias.includes("/") && NATIVE_OPENAI_FAMILY_PATTERN.test(alias)) { + issues.push({ + path: ["alias"], + message: "bare aliases in the OpenAI native family (gpt-*, o1-*, o3-*, o4-*, codex-*) are not allowed", + }); + } + // Cross-namespace collisions: providers, combos, account namespaces, and + // sibling profile aliases all own public model ids that must stay unique. + if (hasOwnProvider(config.providers, alias)) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with configured provider name "${alias}"` }); + } + if (resolveComboId({ combos: config.combos }, alias)) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured combo selector` }); + } + if (alias.includes("/") && codexAccountNamespaceEntries(config).some(([namespace]) => namespace === alias.split("/")[0])) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured codex account namespace` }); + } + for (const [otherId, other] of Object.entries(config.routingProfiles ?? {})) { + if (otherId === id || otherId === options.excludeProfileId) continue; + const otherAlias = typeof other?.alias === "string" ? other.alias.trim() : ""; + if (otherAlias && otherAlias === alias) { + issues.push({ path: ["alias"], message: `alias "${alias}" is already used by profile "${otherId}"` }); + } + } + return issues; +} + +export function routingProfileIssues( + id: string, + raw: unknown, + config: Pick, + options: { excludeProfileId?: string } = {}, +): RoutingProfileValidationIssue[] { + const issues: RoutingProfileValidationIssue[] = []; + if (!isValidPolicyId(id)) { + issues.push({ + path: [], + message: "profile id must start with a letter/number and use letters, numbers, dot, underscore, or hyphen (max 64)", + }); + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + issues.push({ path: [], message: "routing profile must be an object" }); + return issues; + } + const body = raw as Record; + + if (body.alias !== undefined) { + if (typeof body.alias !== "string") { + issues.push({ path: ["alias"], message: "alias must be a string" }); + } else { + const alias = body.alias.trim(); + if (alias) issues.push(...aliasIssues(id, alias, config, options)); + } + } + + if (!Array.isArray(body.candidates) || body.candidates.length === 0) { + issues.push({ path: ["candidates"], message: "candidates must be a non-empty array" }); + } else { + const seen = new Set(); + body.candidates.forEach((rawCandidate, index) => { + if (!rawCandidate || typeof rawCandidate !== "object" || Array.isArray(rawCandidate)) { + issues.push({ path: ["candidates", index], message: `candidates[${index}] must be an object` }); + return; + } + const candidate = rawCandidate as Record; + const provider = typeof candidate.provider === "string" ? candidate.provider.trim() : ""; + const model = typeof candidate.model === "string" ? candidate.model.trim() : ""; + if (!provider) { + issues.push({ path: ["candidates", index, "provider"], message: `candidates[${index}].provider is required` }); + } else if (!hasOwnProvider(config.providers, provider)) { + issues.push({ + path: ["candidates", index, "provider"], + message: `candidates[${index}].provider "${provider}" is not configured`, + }); + } else if (config.providers[provider]?.disabled === true) { + issues.push({ + path: ["candidates", index, "provider"], + message: `candidates[${index}].provider "${provider}" is disabled`, + }); + } + if (!model) { + issues.push({ path: ["candidates", index, "model"], message: `candidates[${index}].model is required` }); + } + if (provider && model) { + const key = `${provider}/${model}`; + if (seen.has(key)) { + issues.push({ path: ["candidates", index], message: `duplicate policy candidate "${key}"` }); + } else { + seen.add(key); + } + } + }); + } + + if (body.require !== undefined) { + if (!body.require || typeof body.require !== "object" || Array.isArray(body.require)) { + issues.push({ path: ["require"], message: "require must be an object" }); + } else { + const require = body.require as Record; + if (require.minContextWindow !== undefined + && (typeof require.minContextWindow !== "number" + || !Number.isInteger(require.minContextWindow) + || require.minContextWindow < 1)) { + issues.push({ path: ["require", "minContextWindow"], message: "minContextWindow must be a positive integer" }); + } + for (const key of ["tools", "imageInput", "structuredOutput", "localOnly", "remoteAllowed", "encryptedCodexTasks"] as const) { + if (require[key] !== undefined && typeof require[key] !== "boolean") { + issues.push({ path: ["require", key], message: `${key} must be a boolean` }); + } + } + if (require.reasoningEffort !== undefined && typeof require.reasoningEffort !== "string") { + issues.push({ path: ["require", "reasoningEffort"], message: "reasoningEffort must be a string" }); + } + if (require.serviceTier !== undefined && typeof require.serviceTier !== "string") { + issues.push({ path: ["require", "serviceTier"], message: "serviceTier must be a string" }); + } + } + } + + if (body.optimize !== undefined) { + if (!body.optimize || typeof body.optimize !== "object" || Array.isArray(body.optimize)) { + issues.push({ path: ["optimize"], message: "optimize must be an object" }); + } else { + const optimize = body.optimize as Record; + for (const key of ["latency", "health", "cost", "quota"] as const) { + if (optimize[key] !== undefined + && (typeof optimize[key] !== "number" + || !Number.isFinite(optimize[key]) + || optimize[key] < 0)) { + issues.push({ path: ["optimize", key], message: `${key} must be a non-negative number` }); + } + } + } + } + + if (body.limits !== undefined) { + if (!body.limits || typeof body.limits !== "object" || Array.isArray(body.limits)) { + issues.push({ path: ["limits"], message: "limits must be an object" }); + } else { + const limits = body.limits as Record; + if (limits.maxEstimatedCostUsd !== undefined + && (typeof limits.maxEstimatedCostUsd !== "number" + || !Number.isFinite(limits.maxEstimatedCostUsd) + || limits.maxEstimatedCostUsd < 0)) { + issues.push({ path: ["limits", "maxEstimatedCostUsd"], message: "maxEstimatedCostUsd must be a non-negative number" }); + } + } + } + + if (body.unknownEvidence !== undefined) { + if (!body.unknownEvidence || typeof body.unknownEvidence !== "object" || Array.isArray(body.unknownEvidence)) { + issues.push({ path: ["unknownEvidence"], message: "unknownEvidence must be an object" }); + } else { + const unknownEvidence = body.unknownEvidence as Record; + for (const key of UNKNOWN_EVIDENCE_KEYS) { + if (unknownEvidence[key] !== undefined + && (unknownEvidence[key] !== "allow" + && unknownEvidence[key] !== "penalize" + && unknownEvidence[key] !== "exclude")) { + issues.push({ path: ["unknownEvidence", key], message: `${key} must be "allow", "penalize", or "exclude"` }); + } + } + } + } + + return issues; +} + +export function routingProfileIssuesForConfig( + config: Pick, +): RoutingProfileValidationIssue[] { + const issues: RoutingProfileValidationIssue[] = []; + for (const [id, raw] of Object.entries(config.routingProfiles ?? {})) { + issues.push(...routingProfileIssues(id, raw, config)); + } + return issues; +} + +function normalizedRequirements(raw: OcxRoutingProfileConfig): NormalizedRoutingProfileRequirements { + const require = raw.require; + if (!require) return {}; + const out: NormalizedRoutingProfileRequirements = {}; + for (const key of REQUIRE_KEYS) { + const value = require[key]; + if (value !== undefined) { + (out as Record)[key] = value; + } + } + return out; +} + +function normalizedUnknownEvidence(raw: OcxRoutingProfileConfig): NormalizedRoutingProfile["unknownEvidence"] { + const configured = raw.unknownEvidence; + const out = { ...DEFAULT_UNKNOWN_EVIDENCE }; + if (configured) { + for (const key of UNKNOWN_EVIDENCE_KEYS) { + const value = configured[key]; + if (value === "allow" || value === "penalize" || value === "exclude") { + out[key] = value; + } + } + } + return out; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const record = value as Record; + const keys = Object.keys(record).sort(); + return `{${keys.map(key => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; +} + +function profileRevision(profile: Omit): string { + const digest = createHash("sha256").update(canonicalJson(profile)).digest("hex"); + return digest.slice(0, 16); +} + +export function normalizeRoutingProfile(id: string, raw: OcxRoutingProfileConfig): NormalizedRoutingProfile { + const alias = typeof raw.alias === "string" ? raw.alias.trim() : ""; + const weights = { ...DEFAULT_PROFILE_WEIGHTS, ...(raw.optimize ?? {}) }; + const weightSum = weights.latency + weights.health + weights.cost + weights.quota; + const safeSum = weightSum > 0 ? weightSum : 1; + const profile: Omit = { + id, + alias: alias || null, + candidates: raw.candidates.map(candidate => ({ + provider: candidate.provider.trim(), + model: candidate.model.trim(), + })), + require: normalizedRequirements(raw), + optimize: { + latency: weights.latency / safeSum, + health: weights.health / safeSum, + cost: weights.cost / safeSum, + quota: weights.quota / safeSum, + }, + limits: { + ...(raw.limits?.maxEstimatedCostUsd !== undefined + ? { maxEstimatedCostUsd: raw.limits.maxEstimatedCostUsd } + : {}), + }, + unknownEvidence: normalizedUnknownEvidence(raw), + }; + return { ...profile, revision: profileRevision(profile) }; +} + +export function getRoutingProfile( + config: { routingProfiles?: Record }, + id: string, +): NormalizedRoutingProfile | undefined { + const profiles = config.routingProfiles; + if (!profiles || !Object.hasOwn(profiles, id)) return undefined; + return normalizeRoutingProfile(id, profiles[id]!); +} + +export function listRoutingProfileIds(config: { routingProfiles?: Record }): string[] { + return Object.keys(config.routingProfiles ?? {}).sort((a, b) => a.localeCompare(b)); +} diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 049d5a0c31..b7b227dcae 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -46,7 +46,7 @@ export interface RouteCapabilityEvidence { image?: Unknownable; structuredOutput?: Unknownable; reasoningEfforts?: string[]; - serviceTier?: Unknownable; + serviceTier?: string | "unknown"; localOnly?: Unknownable; remoteAllowed?: Unknownable; encryptedCodexTasks?: Unknownable; @@ -181,6 +181,7 @@ export interface TraceCandidateInput { accountRef?: string; eligible: boolean; exclusions: RouteExclusionReason[]; + score?: RouteScoreEvidence; } export interface TraceBuildInput { @@ -217,6 +218,7 @@ function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; ex ? { detail: capString(exclusion.detail, budget) } : {}), })), + ...(input.score ? { score: input.score } : {}), }; } @@ -389,8 +391,11 @@ function parseCapability(raw: unknown): RouteCapabilityEvidence | undefined { .slice(0, 8) .map(value => value.slice(0, MAX_TRACE_STRING)); } - const serviceTier = unknownable(raw.serviceTier); - if (serviceTier !== undefined) out.serviceTier = serviceTier; + if (raw.serviceTier === "unknown") { + out.serviceTier = "unknown"; + } else if (typeof raw.serviceTier === "string" && raw.serviceTier) { + out.serviceTier = raw.serviceTier.slice(0, MAX_TRACE_STRING); + } const localOnly = unknownable(raw.localOnly); if (localOnly !== undefined) out.localOnly = localOnly; const remoteAllowed = unknownable(raw.remoteAllowed); diff --git a/src/server/management-api.ts b/src/server/management-api.ts index af8fd499db..61936bd2e4 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -61,6 +61,7 @@ import { handleConfigRoutes } from "./management/config-routes"; import { handleLogsUsageRoutes } from "./management/logs-usage-routes"; import { handleRequestHistoryRoutes } from "./management/request-history-routes"; import { handleRoutingAnalyticsRoutes } from "./management/routing-analytics-routes"; +import { handleRoutingProfileRoutes } from "./management/routing-profile-routes"; import { handleProviderRoutes } from "./management/provider-routes"; import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; @@ -142,6 +143,7 @@ export async function handleManagementAPI( ?? (await handleLogsUsageRoutes(ctx)) ?? (await handleRequestHistoryRoutes(ctx)) ?? (await handleRoutingAnalyticsRoutes(ctx)) + ?? (await handleRoutingProfileRoutes(ctx)) ?? (await handleProviderRoutes(ctx)) ?? (await handleModelRoutes(ctx)) ?? (await handleIntegrationRoutes(ctx)) diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts new file mode 100644 index 0000000000..a9d641d6da --- /dev/null +++ b/src/server/management/routing-profile-routes.ts @@ -0,0 +1,111 @@ +/** + * Routing-profile management API (RI-04). + * + * - `GET /api/routing-profiles` - normalized profiles with revisions + * - `POST /api/routing-profiles/dry-run` - deterministic dry-run evaluation + * (never dispatches an upstream request) + */ + +import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from "../../routing/profile"; +import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; +import { isPlainRecord } from "./shared"; +import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; +import { jsonResponse } from "../auth-cors"; +import type { ManagementContext } from "./context"; + +function profileDto(config: Parameters[0], id: string): Record | null { + const profile = getRoutingProfile(config, id); + if (!profile) return null; + return { + id, + model: policyPublicModelId(id, profile), + revision: profile.revision, + candidates: profile.candidates, + require: profile.require, + optimize: profile.optimize, + limits: profile.limits, + unknownEvidence: profile.unknownEvidence, + }; +} + +function parseEvidence(raw: unknown): { evidence: PolicyRequestEvidence; ok: boolean } { + if (!isPlainRecord(raw)) return { evidence: {}, ok: false }; + const record = raw as Record; + const evidence: PolicyRequestEvidence = {}; + if (typeof record.contextWindow === "number" && Number.isFinite(record.contextWindow) && record.contextWindow >= 0) { + evidence.contextWindow = record.contextWindow; + } + for (const key of ["toolsRequired", "imageInputRequired", "structuredOutputRequired", "encryptedCodexTask"] as const) { + if (typeof record[key] === "boolean") evidence[key] = record[key]; + } + if (typeof record.reasoningEffort === "string") evidence.reasoningEffort = record.reasoningEffort; + if (typeof record.serviceTier === "string") evidence.serviceTier = record.serviceTier; + return { evidence, ok: true }; +} + +function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null { + if (!Array.isArray(raw)) return null; + const out: PolicyCandidateEvidence[] = []; + for (const item of raw) { + if (!isPlainRecord(item)) return null; + const provider = item.provider; + const model = item.model; + if (typeof provider !== "string" || typeof model !== "string") return null; + out.push({ + provider, + model, + ...(typeof item.accountRef === "string" ? { accountRef: item.accountRef } : {}), + // Dry-run evidence is caller-supplied and re-bounded by the trace + // normalizer; structural casts keep the API surface permissive. + ...(isPlainRecord(item.capability) ? { capability: item.capability as unknown as PolicyCandidateEvidence["capability"] } : {}), + ...(isPlainRecord(item.health) ? { health: item.health as unknown as PolicyCandidateEvidence["health"] } : {}), + ...(isPlainRecord(item.quota) ? { quota: item.quota as unknown as PolicyCandidateEvidence["quota"] } : {}), + ...(isPlainRecord(item.cost) ? { cost: item.cost as unknown as PolicyCandidateEvidence["cost"] } : {}), + }); + } + return out; +} + +export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promise { + const { req, url, config } = ctx; + + if (url.pathname === "/api/routing-profiles" && req.method === "GET") { + const profiles = listRoutingProfileIds(config).map(id => profileDto(config, id)).filter( + (profile): profile is Record => profile !== null, + ); + return jsonResponse({ profiles }); + } + + if (url.pathname === "/api/routing-profiles/dry-run" && req.method === "POST") { + let rawBody: unknown; + try { rawBody = await readManagementJsonBody(req); } catch (error) { + rethrowManagementBodyTooLarge(error); + return jsonResponse({ error: "invalid JSON body" }, 400, req, config); + } + if (!isPlainRecord(rawBody)) { + return jsonResponse({ error: "request body must be an object" }, 400, req, config); + } + const body = rawBody as Record; + const profile = typeof body.profile === "string" ? body.profile.trim() : ""; + if (!profile) { + return jsonResponse({ error: { code: "missing_profile", message: "profile is required" } }, 400, req, config); + } + if (!getRoutingProfile(config, profile)) { + return jsonResponse({ error: { code: "unknown_profile", message: `unknown routing profile: ${profile}` } }, 404, req, config); + } + const { evidence, ok } = parseEvidence(body.evidence); + if (!ok) { + return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config); + } + const candidateEvidence = body.candidates === undefined + ? [] + : parseCandidateEvidence(body.candidates); + if (candidateEvidence === null) { + return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config); + } + const result = evaluatePolicyProfile(config, profile, evidence, candidateEvidence); + return jsonResponse(result); + } + + return null; +} diff --git a/src/types.ts b/src/types.ts index ae84aa6740..1774461579 100644 --- a/src/types.ts +++ b/src/types.ts @@ -773,6 +773,13 @@ export interface OcxConfig { }; /** Virtual `combo/` models spanning concrete provider/model targets (issue #133). */ combos?: Record; + /** + * Routing policy profiles (Router Intelligence, RI-04+): explicitly requested + * `policy/` (or configured alias) models select among an explicit + * candidate allowlist using hard capability requirements and deterministic + * scoring. Existing model ids are never routed through profiles implicitly. + */ + routingProfiles?: Record; /** Background proactive token refresh ("Token Guardian"). Off by default; see OcxTokenGuardianConfig. */ tokenGuardian?: OcxTokenGuardianConfig; /** Additional exact origins allowed for CORS (e.g. HTTPS or chrome-extension://). Loopback origins are always allowed. */ @@ -807,6 +814,63 @@ export interface OcxComboConfig { alias?: string; } +export type OcxRoutingUnknownEvidenceMode = "allow" | "penalize" | "exclude"; + +export interface OcxRoutingProfileCandidate { + provider: string; + model: string; +} + +export interface OcxRoutingProfileRequirements { + /** Minimum model context window in tokens. */ + minContextWindow?: number; + tools?: boolean; + imageInput?: boolean; + structuredOutput?: boolean; + reasoningEffort?: string; + serviceTier?: string; + localOnly?: boolean; + remoteAllowed?: boolean; + /** Special encrypted Codex task readability (ChatGPT forward pool). */ + encryptedCodexTasks?: boolean; +} + +export interface OcxRoutingProfileOptimize { + latency?: number; + health?: number; + cost?: number; + quota?: number; +} + +export interface OcxRoutingProfileLimits { + /** Hard per-request estimated-cost ceiling in USD. */ + maxEstimatedCostUsd?: number; +} + +export interface OcxRoutingProfileUnknownEvidence { + capability?: OcxRoutingUnknownEvidenceMode; + health?: OcxRoutingUnknownEvidenceMode; + quota?: OcxRoutingUnknownEvidenceMode; + cost?: OcxRoutingUnknownEvidenceMode; +} + +export interface OcxRoutingProfileConfig { + /** + * Explicit candidate allowlist (`provider/model` refs). No implicit + * expansion in v1. + */ + candidates: OcxRoutingProfileCandidate[]; + /** Optional public model name replacing the default `policy/` slug. */ + alias?: string; + /** Hard requirements evaluated before scoring. */ + require?: OcxRoutingProfileRequirements; + /** Optimization weights; normalized deterministically. */ + optimize?: OcxRoutingProfileOptimize; + limits?: OcxRoutingProfileLimits; + /** How unknown evidence is handled per dimension. */ + unknownEvidence?: OcxRoutingProfileUnknownEvidence; +} + /** * Per-provider proactive-refresh policy. The guardian only ever touches a provider whose EFFECTIVE * policy is "proactive"; "lazy-only" keeps today's on-demand refresh, "disabled" forbids the diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts new file mode 100644 index 0000000000..8de698b140 --- /dev/null +++ b/tests/routing-profile.test.ts @@ -0,0 +1,287 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { validateConfigCandidate } from "../src/config"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + getRoutingProfile, + listRoutingProfileIds, + normalizeRoutingProfile, + parsePolicyModelId, + policyPublicModelId, + resolvePolicyProfileId, + routingProfileIssues, +} from "../src/routing/profile"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-profile-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + }, + combos: { free: { strategy: "failover", targets: [{ provider: "a", model: "m1" }] } }, + codexAccountNamespaces: { work: "acct-1" }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + optimize: { latency: 0.55, health: 0.25, cost: 0.10, quota: 0.10 }, + limits: { maxEstimatedCostUsd: 0.5 }, + unknownEvidence: { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + ...overrides, + }; +} + +describe("routing profiles (RI-04)", () => { + test("normalizes a valid profile with deterministic weights and revision", () => { + const profile = getRoutingProfile(baseConfig(), "fast")!; + expect(profile.id).toBe("fast"); + expect(profile.alias).toBe("ocx/fast"); + expect(profile.candidates).toEqual([ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ]); + expect(profile.require).toMatchObject({ tools: true, minContextWindow: 128000 }); + expect(profile.optimize.latency + profile.optimize.health + profile.optimize.cost + profile.optimize.quota).toBeCloseTo(1); + expect(profile.limits.maxEstimatedCostUsd).toBe(0.5); + expect(profile.revision).toMatch(/^[0-9a-f]{16}$/); + }); + + test("revision digest is stable and changes with the profile", () => { + const config = baseConfig(); + const first = getRoutingProfile(config, "fast")!.revision; + const second = getRoutingProfile(config, "fast")!.revision; + expect(first).toBe(second); + const changed = baseConfig({ + routingProfiles: { + fast: { + ...config.routingProfiles!.fast!, + candidates: [{ provider: "a", model: "m1" }], + }, + }, + }); + expect(getRoutingProfile(changed, "fast")!.revision).not.toBe(first); + }); + + test("weights default and normalize deterministically", () => { + const config = baseConfig({ + routingProfiles: { only: { candidates: [{ provider: "a", model: "m1" }] } }, + }); + const profile = getRoutingProfile(config, "only")!; + expect(profile.optimize).toEqual({ latency: 0.55, health: 0.25, cost: 0.1, quota: 0.1 }); + const weighted = baseConfig({ + routingProfiles: { w: { candidates: [{ provider: "a", model: "m1" }], optimize: { latency: 1, cost: 3 } } }, + }); + const normalized = getRoutingProfile(weighted, "w")!; + // Unspecified weights keep their defaults: latency 1, health 0.25, + // cost 3, quota 0.1 => sum 4.35, normalized deterministically. + expect(normalized.optimize.latency).toBeCloseTo(1 / 4.35); + expect(normalized.optimize.cost).toBeCloseTo(3 / 4.35); + const sum = normalized.optimize.latency + normalized.optimize.health + + normalized.optimize.cost + normalized.optimize.quota; + expect(sum).toBeCloseTo(1); + }); + + test("alias collision validation covers providers, combos, account namespaces, native families", () => { + const config = baseConfig(); + const providerCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "a", + }, config); + expect(providerCollision.some(issue => issue.message.includes("provider name"))).toBe(true); + + const comboCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "combo/free", + }, config); + expect(comboCollision.some(issue => issue.message.includes("reserved"))).toBe(true); + + const nativeCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "gpt-5.6", + }, config); + expect(nativeCollision.some(issue => issue.message.includes("native family"))).toBe(true); + + const siblingCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "ocx/fast", + }, config); + expect(siblingCollision.some(issue => issue.message.includes("already used"))).toBe(true); + }); + + test("candidate validation rejects unconfigured/disabled providers and duplicates", () => { + const config = baseConfig(); + const unconfigured = routingProfileIssues("p", { + candidates: [{ provider: "ghost", model: "m1" }], + }, config); + expect(unconfigured.some(issue => issue.message.includes("not configured"))).toBe(true); + + const disabled = baseConfig({ + providers: { ...baseConfig().providers, c: { adapter: "openai-chat", baseUrl: "https://c.example/v1", apiKey: "kc", models: ["m3"], disabled: true } }, + routingProfiles: { p: { candidates: [{ provider: "c", model: "m3" }] } }, + }); + expect(routingProfileIssues("p", disabled.routingProfiles!.p, disabled).some(issue => issue.message.includes("disabled"))).toBe(true); + + const duplicates = routingProfileIssues("p", { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "a", model: "m1" }, + ], + }, config); + expect(duplicates.some(issue => issue.message.includes("duplicate"))).toBe(true); + }); + + test("config load accepts valid profiles and rejects broken ones", () => { + const valid = validateConfigCandidate(baseConfig()); + expect(valid.ok).toBe(true); + + const broken = validateConfigCandidate(baseConfig({ + routingProfiles: { bad: { candidates: [{ provider: "ghost", model: "m1" }] } }, + })); + expect(broken.ok).toBe(false); + if (!broken.ok) expect(broken.error).toContain("routingProfiles"); + }); + + test("policy id/alias resolution follows canonical-id-first", () => { + const config = baseConfig(); + expect(resolvePolicyProfileId(config, "policy/fast")).toBe("fast"); + expect(resolvePolicyProfileId(config, "ocx/fast")).toBe("fast"); + expect(resolvePolicyProfileId(config, "policy/missing")).toBe("missing"); + expect(resolvePolicyProfileId(config, "unknown")).toBeNull(); + expect(parsePolicyModelId("policy/fast")).toBe("fast"); + expect(parsePolicyModelId("a/m1")).toBeNull(); + expect(policyPublicModelId("fast", getRoutingProfile(config, "fast")!)).toBe("ocx/fast"); + }); + + test("dry-run evaluator: hard requirements gate eligibility", () => { + const config = baseConfig(); + const result = evaluatePolicyProfile(config, "fast", { contextWindow: 200000, toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + { provider: "b", model: "m2", capability: { contextWindow: 64000, tools: true } }, + ]); + expect(result.candidates.length).toBe(2); + expect(result.candidates[0]).toMatchObject({ eligible: true }); + expect(result.candidates[1]).toMatchObject({ eligible: false }); + expect(result.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); + expect(result.selectedIndex).toBe(0); + expect(result.trace.routeKind).toBe("policy"); + expect(result.trace.profile).toEqual({ id: "fast", revision: result.profileRevision }); + expect(result.trace.selected.provider).toBe("a"); + expect(result.trace.selected.model).toBe("m1"); + expect(result.trace.selected.reason).toBe("policy-selected"); + }); + + test("dry-run evaluator: unknown capability follows the profile's unknownEvidence", () => { + const config = baseConfig(); + const unknownEvidence = { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }; + const strict = baseConfig({ + routingProfiles: { strict: { candidates: [{ provider: "a", model: "m1" }], require: { tools: true }, unknownEvidence } }, + providers: baseConfig().providers, + }); + const excluded = evaluatePolicyProfile(strict, "strict", { toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-capability")).toBe(true); + expect(excluded.selectedIndex).toBeNull(); + + const permissive = baseConfig({ + routingProfiles: { permissive: { candidates: [{ provider: "a", model: "m1" }], require: { tools: true }, unknownEvidence: { ...unknownEvidence, capability: "allow" } } }, + }); + const allowed = evaluatePolicyProfile(permissive, "permissive", { toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(allowed.candidates[0]!.eligible).toBe(true); + expect(allowed.selectedIndex).toBe(0); + }); + + test("dry-run evaluator: deterministic tie-break picks the earlier candidate", () => { + const config = baseConfig({ + routingProfiles: { tie: { candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], require: { minContextWindow: 1000 } } }, + }); + const result = evaluatePolicyProfile(config, "tie", { contextWindow: 2000 }, [ + { provider: "a", model: "m1", capability: { contextWindow: 5000 } }, + { provider: "b", model: "m2", capability: { contextWindow: 5000 } }, + ]); + expect(result.selectedIndex).toBe(0); + expect(result.trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + }); + + test("API lists profiles and dry-runs deterministically", async () => { + const config = baseConfig(); + const listReq = new ManagementRequest("http://localhost/api/routing-profiles", { method: "GET" }); + const listResponse = await handleManagementAPI(listReq, new URL(listReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(listResponse).not.toBeNull(); + expect(listResponse!.status).toBe(200); + const listBody = await listResponse!.json() as { profiles?: Array<{ id?: string; revision?: string }> }; + expect(listBody.profiles?.length).toBe(1); + expect(listBody.profiles![0]).toMatchObject({ id: "fast", revision: getRoutingProfile(config, "fast")!.revision }); + + const dryReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "fast", + evidence: { contextWindow: 200000, toolsRequired: true }, + candidates: [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + { provider: "b", model: "m2", capability: { contextWindow: 64000, tools: true } }, + ], + }), + }); + const dryResponse = await handleManagementAPI(dryReq, new URL(dryReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(dryResponse!.status).toBe(200); + const dryBody = await dryResponse!.json() as { selectedIndex?: number | null; trace?: { selected?: { provider?: string } } }; + expect(dryBody.selectedIndex).toBe(0); + expect(dryBody.trace?.selected?.provider).toBe("a"); + }); + + test("API dry-run rejects unknown profiles and invalid evidence", async () => { + const config = baseConfig(); + const unknownReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "nope", evidence: {} }), + }); + const unknownResponse = await handleManagementAPI(unknownReq, new URL(unknownReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(unknownResponse!.status).toBe(404); + + const badReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", evidence: "junk" }), + }); + const badResponse = await handleManagementAPI(badReq, new URL(badReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(badResponse!.status).toBe(400); + }); +}); From 56f17f45c4705762c5783365366962bd5e5bd5e9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:49:56 +0200 Subject: [PATCH 04/10] feat(routing): execute capability-aware policy profiles (RI-05) --- .../001_pr_stack_status.md | 24 ++- src/router.ts | 55 +++++- src/routing/capability.ts | 135 ++++++++++++++ src/routing/evaluator.ts | 26 ++- src/routing/request-evidence.ts | 33 ++++ src/server/chat-completions.ts | 3 +- src/server/claude-messages.ts | 3 +- src/server/responses/core.ts | 5 +- tests/policy-execution.test.ts | 176 ++++++++++++++++++ 9 files changed, 450 insertions(+), 10 deletions(-) create mode 100644 src/routing/capability.ts create mode 100644 src/routing/request-evidence.ts create mode 100644 tests/policy-execution.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 7be9dc428c..379a026dbc 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -43,7 +43,7 @@ other; closing one is a maintainer decision and neither is stale. | RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress | | RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress | | RI-04 | `feat/ri-04-policy-profile-core` | `2069e724e` (RI-03 head) | pending | pending | pending | in progress | -| RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | +| RI-05 | `feat/ri-05-capability-aware-routing` | `00e1c4ae5` (RI-04 head) | pending | pending | pending | in progress | | RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | | RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | @@ -158,3 +158,25 @@ other; closing one is a maintainer decision and neither is stale. - `tests/config.test.ts`: 109/115 pass; the 6 symlink failures reproduce identically on the pristine base (Windows symlink EPERM, environmental) - Remaining Low findings: none + +### RI-05 - feat/ri-05-capability-aware-routing + +- Base SHA: `00e1c4ae5df32cdf6d72957c1a36b334fe4dc0a6` (RI-04 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 2 fixed pre-push - (1) request evidence was wired + but unused by the evaluator - request requirements (`request-tools`, + `request-image-input`) now constrain candidates when the body provably + needs them; (2) trace-score plumbing verified end-to-end (score was added + to trace candidates in RI-04). +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/policy-execution.test.ts`: 8/8 pass - explicit + policy/ + alias execution, precedence unchanged (explicit/combo/ + native/default), all-excluded error, unknown-capability per profile, + request evidence (image/tools) constraints, determinism + - Focused regression suites: 231/231 pass across 8 files (incl. combo + e2e + codex-routing) + - `bun run privacy:scan`: passed +- Remaining Low findings: none diff --git a/src/router.ts b/src/router.ts index 528a6f8f85..5aa354bd9f 100644 --- a/src/router.ts +++ b/src/router.ts @@ -27,6 +27,16 @@ import { type RouteDecisionTraceV1, type TraceCandidateInput, } from "./routing/trace"; +import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; +import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; +import { candidateCapabilityEvidence } from "./routing/capability"; + +export class NoEligiblePolicyCandidateError extends Error { + constructor(readonly profileId: string) { + super(`No eligible candidates for policy profile: ${profileId}`); + this.name = "NoEligiblePolicyCandidateError"; + } +} export interface RouteResult { providerName: string; @@ -420,8 +430,39 @@ function comboRouteCandidates(config: OcxConfig, route: RouteResult): TraceCandi }); } -function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: boolean): RouteResult { +function routeModelInternal( + config: OcxConfig, + modelId: string, + bypassCombos: boolean, + policyEvidence?: PolicyRequestEvidence, +): RouteResult { const slash = modelId.indexOf("/"); + // Policy namespace is system-reserved: an explicit `policy/` or a + // configured profile alias executes the policy evaluator and routes the + // selected candidate. Only explicit requests reach this branch. + const policyId = resolvePolicyProfileId(config, modelId); + if (policyId) { + const profile = getRoutingProfile(config, policyId); + if (!profile) throw new Error(`Unknown routing profile: ${policyId}`); + const candidateEvidence = profile.candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + })); + const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); + if (evaluation.selectedIndex === null) { + throw new NoEligiblePolicyCandidateError(policyId); + } + const selected = evaluation.candidates[evaluation.selectedIndex]!; + const concrete = `${selected.provider}/${selected.model}`; + const routed = routeModelInternal(config, concrete, true, undefined); + return { + ...routed, + routeKind: "policy" as const, + routeReason: "policy-selected", + routeDecision: evaluation.trace, + }; + } if (slash > 0) { const namespace = modelId.slice(0, slash); const binding = codexAccountNamespaceEntries(config) @@ -460,7 +501,7 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo const concrete = `${combo.target.provider}/${combo.target.model}`; // The selected target is already a concrete provider/model reference. Resolve it without // consulting combo aliases again, otherwise an alias that shadows the target can recurse. - const routed = routeModelInternal(config, concrete, true); + const routed = routeModelInternal(config, concrete, true, undefined); return { ...routed, combo, routeKind: "combo" as const, routeReason: "combo-pick" }; } } @@ -535,8 +576,14 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo throw new Error(`No provider configured for model: ${modelId}`); } -export function routeModel(config: OcxConfig, modelId: string): RouteResult { - const route = routeModelInternal(config, modelId, false); +export function routeModel( + config: OcxConfig, + modelId: string, + policyEvidence?: PolicyRequestEvidence, +): RouteResult { + const route = routeModelInternal(config, modelId, false, policyEvidence); + // Policy routes carry a full evaluation trace already; never rebuild it. + if (route.routeDecision) return route; const accountRef = route.codexAccountNamespace; route.routeDecision = buildRouteDecisionTrace({ requestedModel: modelId, diff --git a/src/routing/capability.ts b/src/routing/capability.ts new file mode 100644 index 0000000000..378574886c --- /dev/null +++ b/src/routing/capability.ts @@ -0,0 +1,135 @@ +/** + * Candidate capability evidence for policy routing (RI-05). + * + * Evidence comes from canonical local sources only - provider config maps, + * the provider registry, the cached Codex catalog file, and the native-model + * metadata helpers. No live network fetch happens at routing time. + * + * "Unknown is not zero": any dimension without canonical evidence stays + * `undefined` (unknown) and the profile's `unknownEvidence` policy decides + * how that affects eligibility. + */ + +import type { OcxConfig } from "../types"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; +import { PROVIDER_REGISTRY } from "../providers/registry"; +import { + nativeInputModalities, + nativeOpenAiContextWindow, + nativeParallelToolCalls, + nativeReasoningEfforts, +} from "../codex/catalog/metadata"; +import { readCatalog, readCodexCatalogPath } from "../codex/catalog/parsing"; +import type { RouteCapabilityEvidence } from "./trace"; + +function cachedCatalogModels(): Array<{ provider: string; id: string; contextWindow?: number; inputModalities?: string[]; reasoningEfforts?: string[]; capabilities?: string[] }> { + try { + const catalog = readCatalog(readCodexCatalogPath()); + const models = catalog?.models; + if (!Array.isArray(models)) return []; + return models + .filter((model): model is Record & { id: string; provider: string } => + typeof model === "object" && model !== null && typeof model.id === "string" && typeof model.provider === "string") + .map(model => ({ + provider: model.provider, + id: model.id, + ...(typeof model.contextWindow === "number" ? { contextWindow: model.contextWindow } : {}), + ...(Array.isArray(model.inputModalities) + ? { inputModalities: model.inputModalities.filter((value): value is string => typeof value === "string") } + : {}), + ...(Array.isArray(model.reasoningEfforts) + ? { reasoningEfforts: model.reasoningEfforts.filter((value): value is string => typeof value === "string") } + : {}), + ...(Array.isArray(model.capabilities) + ? { capabilities: model.capabilities.filter((value): value is string => typeof value === "string") } + : {}), + })); + } catch { + return []; + } +} + +function isLocalHostname(hostname: string): boolean { + const normalized = hostname.trim().toLowerCase().replace(/\.$/, ""); + return normalized === "localhost" || normalized === "127.0.0.1" + || normalized === "::1" || normalized === "[::1]" + || normalized.endsWith(".localhost"); +} + +function isPrivateHostname(hostname: string): boolean { + return hostname.startsWith("10.") || hostname.startsWith("192.168.") + || /^172\.(1[6-9]|2\d|3[01])\./.test(hostname); +} + +function localRemoteEvidence(baseUrl: string | undefined): Pick { + if (typeof baseUrl !== "string" || baseUrl.length === 0) return {}; + try { + const hostname = new URL(baseUrl).hostname; + if (!hostname) return {}; + if (isLocalHostname(hostname) || isPrivateHostname(hostname)) return { localOnly: true }; + return { remoteAllowed: true }; + } catch { + return {}; + } +} + +/** + * Assemble canonical capability evidence for one `provider/model` candidate. + * Sources (in priority order): provider config maps, provider registry hints, + * cached Codex catalog row, native-model metadata. + */ +export function candidateCapabilityEvidence( + config: OcxConfig, + providerName: string, + modelId: string, +): RouteCapabilityEvidence { + const provider = config.providers[providerName]; + const registryEntry = PROVIDER_REGISTRY.find(entry => entry.id === providerName); + const catalogRow = cachedCatalogModels().find(model => model.provider === providerName && model.id === modelId); + const isNative = providerName === "openai" && !modelId.includes("/"); + + const contextWindow = provider?.modelContextWindows?.[modelId] + ?? provider?.contextWindow + ?? registryEntry?.modelContextWindows?.[modelId] + ?? catalogRow?.contextWindow + ?? (isNative ? nativeOpenAiContextWindow(modelId) : undefined); + + const modalities = provider?.modelInputModalities?.[modelId] + ?? registryEntry?.modelInputModalities?.[modelId] + ?? catalogRow?.inputModalities + ?? (isNative ? nativeInputModalities(modelId) : undefined); + const image = Array.isArray(modalities) + ? modalities.includes("image") + : undefined; + + const capabilities = catalogRow?.capabilities ?? []; + const tools = capabilities.includes("tools") + || (isNative ? true : provider?.parallelToolCalls === true) + || undefined; + + const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId] + ?? registryEntry?.modelReasoningEfforts?.[modelId] + ?? catalogRow?.reasoningEfforts + ?? (isNative ? nativeReasoningEfforts(modelId) : undefined); + + const tierSupport = provider?.supportsServiceTier + ?? registryEntry?.supportsServiceTier; + const serviceTier = tierSupport === true + ? "supported" + : tierSupport === false ? "unsupported" : "unknown"; + + const localRemote = localRemoteEvidence(provider?.baseUrl); + const encryptedCodexTasks = isCanonicalOpenAiForwardProvider( + provider ?? { adapter: "", authMode: undefined, baseUrl: undefined }, + ); + + return { + ...(typeof contextWindow === "number" ? { contextWindow } : {}), + ...(typeof image === "boolean" ? { image } : {}), + ...(typeof tools === "boolean" ? { tools } : {}), + ...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}), + ...(serviceTier !== "unknown" ? { serviceTier } : {}), + ...localRemote, + encryptedCodexTasks, + }; +} diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 3e550dc178..bddff00300 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -144,6 +144,27 @@ function requirementFor( return requirements; } +/** + * Request-side capability constraints (RI-05): a request that provably needs + * tools or image input imposes the same requirement on candidates as the + * profile's hard requirements. Undefined request evidence adds nothing. + */ +function requestRequirements( + requestEvidence: PolicyRequestEvidence, + capability: RouteCapabilityEvidence | undefined, +): RouteRequirementEvidence[] { + const requirements: RouteRequirementEvidence[] = []; + if (requestEvidence.toolsRequired === true) { + const tools = booleanRequirement("request-tools", true, capability?.tools); + if (tools) requirements.push(tools); + } + if (requestEvidence.imageInputRequired === true) { + const image = booleanRequirement("request-image-input", true, capability?.image); + if (image) requirements.push(image); + } + return requirements; +} + function unsatisfiedOrUnknown(requirements: RouteRequirementEvidence[]): RouteRequirementEvidence[] { return requirements.filter(requirement => requirement.outcome !== "satisfied"); } @@ -173,7 +194,10 @@ export function evaluatePolicyProfile( const evidence = candidateEvidence.find( candidate => candidate.provider === declared.provider && candidate.model === declared.model, ) ?? { provider: declared.provider, model: declared.model }; - const requirements = requirementFor(profile.require, evidence.capability); + const requirements = [ + ...requirementFor(profile.require, evidence.capability), + ...requestRequirements(requestEvidence, evidence.capability), + ]; const exclusions: RouteExclusionReason[] = []; const bad = unsatisfiedOrUnknown(requirements); for (const requirement of bad) { diff --git a/src/routing/request-evidence.ts b/src/routing/request-evidence.ts new file mode 100644 index 0000000000..66fe34fc41 --- /dev/null +++ b/src/routing/request-evidence.ts @@ -0,0 +1,33 @@ +/** + * Cheap request-side evidence extraction for policy routing (RI-05). + * + * Extracts only what the request body can prove: whether the caller asked for + * tools and whether the input contains image parts. Context-window size is + * left unknown at routing time (documented limitation) - the dry-run API/CLI + * remains the evidence-inspection surface for context-sensitive profiles. + */ + +import type { PolicyRequestEvidence } from "./evaluator"; + +function inputContainsImage(input: unknown): boolean { + if (typeof input === "string") return false; + if (!Array.isArray(input)) return false; + return input.some(part => { + if (!part || typeof part !== "object" || Array.isArray(part)) return false; + const record = part as Record; + if (record.type === "image" || record.type === "input_image") return true; + if (record.image_url !== undefined || record.image !== undefined) return true; + return false; + }); +} + +export function evidenceFromBody(body: unknown): PolicyRequestEvidence { + if (!body || typeof body !== "object" || Array.isArray(body)) return {}; + const record = body as Record; + const tools = Array.isArray(record.tools) && record.tools.length > 0; + const image = inputContainsImage(record.input); + return { + ...(tools ? { toolsRequired: true } : {}), + ...(image ? { imageInputRequired: true } : {}), + }; +} diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index 0f352a249f..b4d85b4999 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -19,6 +19,7 @@ import { redactSecretString } from "../lib/redact"; import { resolveClientRetryAfter } from "../lib/retry-after"; import { estimateTokens } from "../lib/token-estimate"; import { routeModel } from "../router"; +import { evidenceFromBody } from "../routing/request-evidence"; import { resolveWireProtocolOverride } from "./adapter-resolve"; import type { OcxConfig } from "../types"; import { readJsonRequestBody } from "./request-decompress"; @@ -110,7 +111,7 @@ async function handleChatCompletionsWithBudget( let nativeRoute = false; let directRoute = false; try { - const route = routeModel(config, internalBody.model as string); + const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody)); // Settle the wire once so every branch below reads the adapter this model will // actually use, not the provider-wide default (#404). route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "chat"); diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index d0b5a045b6..9b5ffcee88 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -26,6 +26,7 @@ import { import { clearableDeadline, idleDeadline } from "../lib/abort"; import { estimateTokens } from "../lib/token-estimate"; import { routeModel } from "../router"; +import { evidenceFromBody } from "../routing/request-evidence"; import { resolveWireProtocolOverride } from "./adapter-resolve"; import type { OcxConfig } from "../types"; import { readJsonRequestBody } from "./request-decompress"; @@ -628,7 +629,7 @@ async function handleClaudeMessagesWithBudget( // verified live 2026-07-11). Strip them for that route; routed providers keep them. let nativeRoute = false; try { - const route = routeModel(config, internalBody.model as string); + const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody)); // Settle the wire once so the sampling decision below reads the effective // adapter rather than the provider-wide default (#404). route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "anthropic"); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index e6cd447b90..9495d192bf 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -17,6 +17,7 @@ import { rememberResponseState, } from "../../responses/state"; import { routeModel, type RouteResult } from "../../router"; +import { evidenceFromBody } from "../../routing/request-evidence"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -1372,7 +1373,7 @@ async function handleResponsesInner( let route: RouteResult; try { - route = routeModel(config, parsed.modelId); + route = routeModel(config, parsed.modelId, evidenceFromBody(parsed._rawBody)); } catch (err) { if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); @@ -1444,7 +1445,7 @@ async function handleResponsesInner( if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { try { - route = routeModel(config, fallback.to); + route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody)); logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts new file mode 100644 index 0000000000..00c0f67d39 --- /dev/null +++ b/tests/policy-execution.test.ts @@ -0,0 +1,176 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import { getRoutingProfile } from "../src/routing/profile"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-policy-exec-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://a.example/v1", + apiKey: "ka", + models: ["m1"], + modelContextWindows: { m1: 200_000 }, + modelInputModalities: { m1: ["text", "image"] }, + parallelToolCalls: true, + }, + b: { + adapter: "openai-chat", + baseUrl: "https://b.example/v1", + apiKey: "kb", + models: ["m2"], + modelContextWindows: { m2: 64_000 }, + modelInputModalities: { m2: ["text"] }, + }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + combos: { + free: { + strategy: "failover", + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + unknownEvidence: { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + ...overrides, + }; +} + +describe("policy execution (RI-05)", () => { + test("explicit policy/ request executes the evaluator and routes the winner", () => { + const config = baseConfig(); + const route = routeModel(config, "policy/fast"); + expect(route.routeKind).toBe("policy"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + const trace = route.routeDecision!; + expect(trace.routeKind).toBe("policy"); + expect(trace.profile).toEqual({ id: "fast", revision: getRoutingProfile(config, "fast")!.revision }); + expect(trace.candidates.length).toBe(2); + expect(trace.candidates[0]).toMatchObject({ provider: "a", model: "m1", eligible: true }); + expect(trace.candidates[1]).toMatchObject({ provider: "b", model: "m2", eligible: false }); + expect(trace.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); + expect(trace.selected.provider).toBe("a"); + expect(trace.selected.model).toBe("m1"); + expect(trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + }); + + test("profile alias executes the same policy", () => { + const route = routeModel(baseConfig(), "ocx/fast"); + expect(route.routeKind).toBe("policy"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + }); + + test("existing explicit, combo, native and default routes are unchanged", () => { + const config = baseConfig(); + expect(routeModel(config, "a/m1")).toMatchObject({ routeKind: "explicit-provider", providerName: "a", modelId: "m1" }); + expect(routeModel(config, "combo/free")).toMatchObject({ routeKind: "combo", combo: { comboId: "free" } }); + expect(routeModel(config, "gpt-5.6")).toMatchObject({ routeKind: "native", providerName: "openai", modelId: "gpt-5.6" }); + expect(routeModel(config, "totally-unknown")).toMatchObject({ routeKind: "default-provider", providerName: "a" }); + }); + + test("all candidates excluded throws NoEligiblePolicyCandidateError", () => { + const config = baseConfig({ + routingProfiles: { + strict: { + candidates: [{ provider: "b", model: "m2" }], + require: { minContextWindow: 128000 }, + }, + }, + }); + expect(() => routeModel(config, "policy/strict")).toThrow(NoEligiblePolicyCandidateError); + }); + + test("unknown capability follows the profile unknownEvidence (exclude default)", () => { + // Provider "b" has no parallelToolCalls and no catalog row: tools unknown. + const config = baseConfig({ + routingProfiles: { + toolsOnly: { + candidates: [{ provider: "b", model: "m2" }], + require: { tools: true }, + }, + }, + }); + expect(() => routeModel(config, "policy/toolsOnly")).toThrow(NoEligiblePolicyCandidateError); + + const permissive = baseConfig({ + routingProfiles: { + toolsOnly: { + candidates: [{ provider: "b", model: "m2" }], + require: { tools: true }, + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const route = routeModel(permissive, "policy/toolsOnly"); + expect(route.providerName).toBe("b"); + expect(route.modelId).toBe("m2"); + }); + + test("request evidence constrains candidates: image input excludes non-image models", () => { + const config = baseConfig({ + routingProfiles: { + image: { candidates: [{ provider: "b", model: "m2" }] }, + }, + }); + // No image in the request: the request requirement is absent; b is eligible. + const plain = routeModel(config, "policy/image"); + expect(plain.providerName).toBe("b"); + // Image request: b's modalities are text-only -> excluded. + expect(() => routeModel(config, "policy/image", { imageInputRequired: true })).toThrow(NoEligiblePolicyCandidateError); + }); + + test("request tools requirement is enforced when provably needed", () => { + const config = baseConfig({ + routingProfiles: { + tools: { candidates: [{ provider: "b", model: "m2" }] }, + }, + }); + expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "b", modelId: "m2" }); + // b's tools support is unknown -> request requiring tools excludes it. + expect(() => routeModel(config, "policy/tools", { toolsRequired: true })).toThrow(NoEligiblePolicyCandidateError); + }); + + test("policy selection is deterministic across calls", () => { + const config = baseConfig(); + const first = routeModel(config, "policy/fast"); + const second = routeModel(config, "policy/fast"); + expect(first.providerName).toBe(second.providerName); + expect(first.modelId).toBe(second.modelId); + expect(first.routeDecision!.selected).toEqual(second.routeDecision!.selected); + }); +}); From 909ce21d4ffe4dbcad9c9baa2efb1c94e1e7dcd6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:58:37 +0200 Subject: [PATCH 05/10] feat(routing): add evidence-based route health scoring (RI-06) --- .../001_pr_stack_status.md | 37 ++- src/router.ts | 5 + src/routing/evaluator.ts | 34 ++- src/routing/health.ts | 200 ++++++++++++++++ src/routing/history/indexer.ts | 9 +- src/routing/trace.ts | 8 + tests/health-scoring.test.ts | 222 ++++++++++++++++++ tests/policy-execution.test.ts | 8 +- tests/routing-profile.test.ts | 4 +- 9 files changed, 521 insertions(+), 6 deletions(-) create mode 100644 src/routing/health.ts create mode 100644 tests/health-scoring.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 379a026dbc..b15e3db8f9 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -44,7 +44,7 @@ other; closing one is a maintainer decision and neither is stale. | RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress | | RI-04 | `feat/ri-04-policy-profile-core` | `2069e724e` (RI-03 head) | pending | pending | pending | in progress | | RI-05 | `feat/ri-05-capability-aware-routing` | `00e1c4ae5` (RI-04 head) | pending | pending | pending | in progress | -| RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | +| RI-06 | `feat/ri-06-health-aware-routing` | `56f17f45c` (RI-05 head) | pending | pending | pending | in progress | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | | RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | | RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued | @@ -180,3 +180,38 @@ other; closing one is a maintainer decision and neither is stale. e2e + codex-routing) - `bun run privacy:scan`: passed - Remaining Low findings: none + +### RI-06 - feat/ri-06-health-aware-routing + +- Base SHA: `56f17f45c4705762c5783365366962bd5e5bd5e9` (RI-05 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 4 fixed pre-push - + 1. route-time health evidence needed synchronous index access; the indexer + refresh is now a sync core (`openRequestHistoryIndexSync`) with the async + single-flight wrapper around it; + 2. unknown-health "penalize" now folds a deterministic floor (0.3) into the + score instead of silently skipping the component; + 3. trace candidates now carry capability/health/quota/cost evidence; + 4. score assertions in RI-04/RI-05 tests updated for the new health + component (behavioral change by design). +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/health-scoring.test.ts`: 9/9 pass - historical + evidence (success rate, consecutive failures, latency, samples), + cancellation/invalid-request neutrality, incomplete streams, low-sample + confidence, hard-cooldown authority + exclusion, unknown-health policy, + score component + trace evidence, health-driven selection, execution path + - Focused regression suites: 196/196 pass across 8 files + - `bun run privacy:scan`: passed +- Remaining Low findings: none + +## Baseline note + +The full-suite baseline on this Windows machine did not complete within the +available window (background run, >3h, no summary emitted; the suite is +~8k tests and this machine is heavily loaded). Focused suites, typecheck and +privacy:scan pass per PR; the upstream PR #966 verification report records +~7941 pass / 10 environmental failures on clean dev. A final full-suite +attempt is scheduled at stack end. diff --git a/src/router.ts b/src/router.ts index 5aa354bd9f..fb4272ac21 100644 --- a/src/router.ts +++ b/src/router.ts @@ -30,6 +30,7 @@ import { import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; import { candidateCapabilityEvidence } from "./routing/capability"; +import { healthEvidenceForCandidate } from "./routing/health"; export class NoEligiblePolicyCandidateError extends Error { constructor(readonly profileId: string) { @@ -448,6 +449,10 @@ function routeModelInternal( provider: candidate.provider, model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + health: healthEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + }), })); const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); if (evaluation.selectedIndex === null) { diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index bddff00300..0237650eb2 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -20,6 +20,10 @@ import { type Unknownable, } from "./trace"; import { getRoutingProfile, type NormalizedRoutingProfile } from "./profile"; +import { healthScore } from "./health"; + +/** Unknown health under "penalize": a low-but-not-zero deterministic floor. */ +export const HEALTH_UNKNOWN_PENALTY_SCORE = 0.3; export interface PolicyRequestEvidence { /** Required context window for this request (tokens). */ @@ -213,12 +217,35 @@ export function evaluatePolicyProfile( // or allow. "penalize" currently cannot move the score because RI-04 has // no capability component yet - the capability score arrives with RI-05. const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; - const eligible = !unsatisfied && !excludedByUnknown; + let eligible = !unsatisfied && !excludedByUnknown; + + // Health scoring (RI-06): live hard cooldown is authoritative and + // excludes; unknown health follows the profile's unknownEvidence policy; + // historical health never overrides explicit ineligibility. + const health = evidence.health; + let healthValue = health ? healthScore(health) : null; + if (health?.cooldownUntilMs !== undefined && health.cooldownUntilMs > Date.now()) { + exclusions.push({ code: "cooldown" }); + eligible = false; + } else if (healthValue === null && profile.unknownEvidence.health === "exclude") { + exclusions.push({ code: "unknown-health" }); + eligible = false; + } else if (healthValue === null && profile.unknownEvidence.health === "penalize") { + healthValue = HEALTH_UNKNOWN_PENALTY_SCORE; + } const score: RouteScoreEvidence = { total: configuredPriorityScore(index, profile.candidates.length), components: { configuredPriority: configuredPriorityScore(index, profile.candidates.length) }, }; + const healthWeight = profile.optimize.health; + if (healthWeight > 0 && healthValue !== null) { + const priority = configuredPriorityScore(index, profile.candidates.length); + const total = priority * (1 - healthWeight) + healthValue * healthWeight; + score.total = total; + score.components.health = healthValue; + score.components.configuredPriority = priority; + } const evaluated: PolicyEvaluationCandidate = { provider: evidence.provider, model: evidence.model, @@ -230,6 +257,7 @@ export function evaluatePolicyProfile( ...(evidence.health ? { health: evidence.health } : {}), ...(evidence.quota ? { quota: evidence.quota } : {}), ...(evidence.cost ? { cost: evidence.cost } : {}), + ...(evidence.health ? { health: evidence.health } : {}), score, }; candidates.push(evaluated); @@ -252,6 +280,10 @@ export function evaluatePolicyProfile( eligible: candidate.eligible, exclusions: candidate.exclusions, ...(candidate.score ? { score: candidate.score } : {}), + ...(candidate.capability ? { capability: candidate.capability } : {}), + ...(candidate.health ? { health: candidate.health } : {}), + ...(candidate.quota ? { quota: candidate.quota } : {}), + ...(candidate.cost ? { cost: candidate.cost } : {}), })), selected: selectedIndex === null ? { provider: candidates[0]?.provider ?? "", model: candidates[0]?.model ?? "", reason: "no-eligible-candidate" } diff --git a/src/routing/health.ts b/src/routing/health.ts new file mode 100644 index 0000000000..0656efb2b9 --- /dev/null +++ b/src/routing/health.ts @@ -0,0 +1,200 @@ +/** + * Evidence-based route health (RI-06). + * + * Health evidence combines: + * - live in-memory routing state: Codex account cooldown / soft-avoid + * (authoritative hard state); + * - historical evidence from the request-history index: success rate, + * consecutive failures, incomplete-stream rate, recent latency, sample + * count, recency-decayed weights. + * + * Failure classification is strict: client cancellations, invalid requests + * (4xx except quota 429) and synthetic policy refusals never damage target + * health. Transport-neutral failures are excluded by the classification the + * routing layer already records (host/account split per #914 work). + * + * All formulas are deterministic with documented constants; no ML. + */ + +import type { OcxConfig } from "../types"; +import { openRequestHistoryIndexSync, requestHistoryDb } from "./history/indexer"; +import { + getCodexAccountCooldownUntil, + getCodexAccountSoftAvoidUntil, + isCodexAccountInCooldown, +} from "../codex/routing"; +import type { RouteHealthEvidence } from "./trace"; + +export const HEALTH_SCORE_CONSTANTS = { + /** Recent-success weight in the composite. */ + SUCCESS_WEIGHT: 0.50, + /** Incomplete-stream (negative) weight. */ + INCOMPLETE_WEIGHT: 0.15, + /** Recent-latency weight. */ + LATENCY_WEIGHT: 0.20, + /** Consecutive-failure recovery weight. */ + RECOVERY_WEIGHT: 0.15, + /** p50 latency at or above this (ms) scores zero on the latency axis. */ + LATENCY_TARGET_MS: 60_000, + /** Samples needed for full confidence; fewer samples scale the score down. */ + MIN_CONFIDENCE_SAMPLES: 20, + /** Soft-avoid multiplies the composite. */ + SOFT_AVOID_MULTIPLIER: 0.5, + /** Recency decay: a sample loses half its weight every RECENCY_HALF_LIFE_DAYS. */ + RECENCY_HALF_LIFE_DAYS: 7, +} as const; + +export const HEALTH_WINDOW_MS = 14 * 86_400_000; +export const HEALTH_MAX_SAMPLES = 100; + +export interface HealthEvidenceInput { + provider: string; + model: string; + accountRef?: string; + /** Live codex account id for cooldown/soft-avoid state (provider "openai"). */ + codexAccountId?: string; + now?: number; +} + +interface HealthSample { + status: number; + closeReason: string | null; + terminalStatus: string | null; + durationMs: number; + timestamp: number; +} + +function classifySample(sample: HealthSample): "success" | "failure" | "neutral" { + if (sample.closeReason === "client_cancel" || sample.status === 499) return "neutral"; + // Invalid requests and policy refusals must not poison target health. + if (sample.status >= 400 && sample.status < 500 && sample.status !== 429) return "neutral"; + if (sample.terminalStatus === "incomplete") return "failure"; + if (sample.terminalStatus && sample.terminalStatus !== "completed") return "failure"; + if (sample.status >= 400) return "failure"; + return "success"; +} + +function decayWeight(timestamp: number, now: number): number { + const ageDays = Math.max(0, now - timestamp) / 86_400_000; + return Math.pow(0.5, ageDays / HEALTH_SCORE_CONSTANTS.RECENCY_HALF_LIFE_DAYS); +} + +function median(sorted: number[]): number | undefined { + if (sorted.length === 0) return undefined; + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; +} + +/** + * Historical health evidence from the derived index (synchronous: called at + * routing time). Never throws; an unopened/unreadable index yields unknown. + */ +export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHealthEvidence { + const now = input.now ?? Date.now(); + const evidence: RouteHealthEvidence = {}; + + // Live authoritative state: hard cooldown and soft-avoid for Codex pool + // accounts. Cooldown stays authoritative over any historical score. + if (input.codexAccountId && input.provider === "openai") { + if (isCodexAccountInCooldown(input.codexAccountId, now)) { + const until = getCodexAccountCooldownUntil(input.codexAccountId, now); + if (until !== null) evidence.cooldownUntilMs = until; + } + const softAvoidUntil = getCodexAccountSoftAvoidUntil(input.codexAccountId, now); + if (softAvoidUntil !== null && softAvoidUntil > now) evidence.softAvoidUntilMs = softAvoidUntil; + } + + try { + openRequestHistoryIndexSync(); + const handle = requestHistoryDb(); + const where: string[] = ["provider = ?", "model = ?", "timestamp >= ?"]; + const values: Array = [input.provider, input.model, now - HEALTH_WINDOW_MS]; + if (input.accountRef) { + where.push("api_key_id = ?"); + values.push(input.accountRef); + } + const rows = handle.query( + `SELECT status, close_reason AS closeReason, terminal_status AS terminalStatus, + duration_ms AS durationMs, timestamp + FROM requests WHERE ${where.join(" AND ")} + ORDER BY timestamp DESC LIMIT ?`, + ).all(...values, HEALTH_MAX_SAMPLES) as HealthSample[]; + + let successes = 0; + let failures = 0; + let incompleteStreams = 0; + let weightedSuccess = 0; + let weightedTotal = 0; + const latencies: number[] = []; + let consecutiveFailures = 0; + for (let index = 0; index < rows.length; index++) { + const row = rows[index]!; + const kind = classifySample(row); + const weight = decayWeight(row.timestamp, now); + if (kind === "neutral") continue; + if (kind === "success") { + successes += 1; + weightedSuccess += weight; + weightedTotal += weight; + } else { + failures += 1; + weightedTotal += weight; + } + if (row.terminalStatus === "incomplete") incompleteStreams += 1; + latencies.push(row.durationMs); + } + // Consecutive failures: walk newest -> oldest until a success. + let consecutive = 0; + for (const row of rows) { + const kind = classifySample(row); + if (kind === "neutral") continue; + if (kind === "failure") consecutive += 1; + else break; + } + consecutiveFailures = consecutive; + + const sampleCount = successes + failures; + if (sampleCount > 0) { + evidence.sampleCount = sampleCount; + evidence.successRate = weightedTotal > 0 ? weightedSuccess / weightedTotal : 0; + if (consecutiveFailures > 0) evidence.failures = consecutiveFailures; + if (incompleteStreams > 0) evidence.incompleteStreamRate = incompleteStreams / sampleCount; + latencies.sort((a, b) => a - b); + const p50 = median(latencies); + if (p50 !== undefined) evidence.recentLatencyMs = p50; + evidence.recencyWeight = decayWeight(rows[0]!.timestamp, now); + } + } catch { + /* index unreadable: evidence stays unknown */ + } + + return evidence; +} + +/** + * Deterministic health score in [0,1]. Returns null when evidence is unknown + * (no samples) so callers can apply the profile's unknownEvidence policy. + * A live hard cooldown scores 0 (authoritative). + */ +export function healthScore(evidence: RouteHealthEvidence | undefined, now = Date.now()): number | null { + if (!evidence) return null; + if (evidence.cooldownUntilMs !== undefined && evidence.cooldownUntilMs > now) return 0; + if (!evidence.sampleCount || evidence.sampleCount < 1) return null; + const successRate = evidence.successRate ?? 0; + const incompleteRate = evidence.incompleteStreamRate ?? 0; + const p50 = evidence.recentLatencyMs; + const latencyScore = p50 === undefined + ? 0.5 + : Math.max(0, Math.min(1, 1 - p50 / HEALTH_SCORE_CONSTANTS.LATENCY_TARGET_MS)); + const consecutive = evidence.failures ?? 0; + const recoveryScore = 1 - Math.min(1, consecutive / 5); + const composite = HEALTH_SCORE_CONSTANTS.SUCCESS_WEIGHT * successRate + + HEALTH_SCORE_CONSTANTS.INCOMPLETE_WEIGHT * (1 - incompleteRate) + + HEALTH_SCORE_CONSTANTS.LATENCY_WEIGHT * latencyScore + + HEALTH_SCORE_CONSTANTS.RECOVERY_WEIGHT * recoveryScore; + const confidence = Math.min(1, evidence.sampleCount / HEALTH_SCORE_CONSTANTS.MIN_CONFIDENCE_SAMPLES); + const softAvoid = evidence.softAvoidUntilMs !== undefined && evidence.softAvoidUntilMs > now + ? HEALTH_SCORE_CONSTANTS.SOFT_AVOID_MULTIPLIER + : 1; + return composite * confidence * softAvoid; +} diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index 287c876b5c..bd0e56489f 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -393,7 +393,7 @@ function fullRebuild(dbHandle: Database, reason: string): void { setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); } -async function refreshLocked(): Promise { +function refreshLockedSync(): RequestHistoryIndexMeta { openIndexDb(); const state = ensureSchemaAndIdentity(db!); const handle = db!; @@ -420,6 +420,11 @@ async function refreshLocked(): Promise { return metaFor(handle); } +/** Synchronous refresh for routing-time evidence reads (RI-06+). */ +export function openRequestHistoryIndexSync(): RequestHistoryIndexMeta { + return refreshLockedSync(); +} + /** * Open (and refresh) the index. Single-flight: concurrent callers share one * refresh. Never throws for missing/corrupt index or ledger state; those are @@ -427,7 +432,7 @@ async function refreshLocked(): Promise { */ export function openRequestHistoryIndex(): Promise { if (!openPromise) { - openPromise = refreshLocked().finally(() => { + openPromise = Promise.resolve(refreshLockedSync()).finally(() => { openPromise = null; }); } diff --git a/src/routing/trace.ts b/src/routing/trace.ts index b7b227dcae..9368102f45 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -182,6 +182,10 @@ export interface TraceCandidateInput { eligible: boolean; exclusions: RouteExclusionReason[]; score?: RouteScoreEvidence; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; } export interface TraceBuildInput { @@ -219,6 +223,10 @@ function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; ex : {}), })), ...(input.score ? { score: input.score } : {}), + ...(input.capability ? { capability: input.capability } : {}), + ...(input.health ? { health: input.health } : {}), + ...(input.quota ? { quota: input.quota } : {}), + ...(input.cost ? { cost: input.cost } : {}), }; } diff --git a/tests/health-scoring.test.ts b/tests/health-scoring.test.ts new file mode 100644 index 0000000000..bcc4c76ab0 --- /dev/null +++ b/tests/health-scoring.test.ts @@ -0,0 +1,222 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendUsageEntry, resetUsageReadCacheForTests, type PersistedUsageEntry } from "../src/usage/log"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; +import { healthEvidenceForCandidate, healthScore, HEALTH_SCORE_CONSTANTS } from "../src/routing/health"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +function row( + requestId: string, + status: number, + durationMs: number, + overrides: Partial = {}, +): PersistedUsageEntry { + return { + requestId, + timestamp: Date.now() - 60_000, + provider: "a", + model: "m1", + status, + durationMs, + usageStatus: "reported", + ...overrides, + }; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-health-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + }, + routingProfiles: { + healthy: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + ...overrides, + }; +} + +describe("health-aware scoring (RI-06)", () => { + test("historical evidence derives success rate, consecutive failures, latency, samples", async () => { + for (let index = 0; index < 23; index++) { + appendUsageEntry(row(`ok-${index}`, 200, 1000 + index)); + } + appendUsageEntry(row("fail-1", 503, 4000, { timestamp: Date.now() - 5_000 })); + appendUsageEntry(row("fail-2", 503, 4000, { timestamp: Date.now() - 4_000 })); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(25); + expect(evidence.failures).toBe(2); + expect(evidence.successRate).toBeCloseTo(23 / 25, 2); + expect(evidence.recentLatencyMs).toBeDefined(); + expect(evidence.incompleteStreamRate).toBeUndefined(); + const score = healthScore(evidence)!; + expect(score).toBeGreaterThan(0.5); + }); + + test("client cancellations and invalid requests never damage health", async () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 1000)); + appendUsageEntry(row("cancel", 499, 500, { closeReason: "client_cancel" })); + appendUsageEntry(row("invalid", 400, 500, { closeReason: "non_stream" })); + appendUsageEntry(row("refused", 404, 500)); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(5); + expect(evidence.successRate).toBe(1); + }); + + test("incomplete streams lower the health score", async () => { + for (let index = 0; index < 10; index++) appendUsageEntry(row(`ok-${index}`, 200, 1000)); + appendUsageEntry(row("inc", 200, 1000, { terminalStatus: "incomplete" })); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.incompleteStreamRate).toBeCloseTo(1 / 11, 2); + const score = healthScore(evidence)!; + expect(score).toBeLessThan(0.99); + }); + + test("low sample counts reduce confidence", async () => { + appendUsageEntry(row("only", 200, 1000)); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const score = healthScore(evidence)!; + const full = healthScore({ ...evidence, sampleCount: HEALTH_SCORE_CONSTANTS.MIN_CONFIDENCE_SAMPLES })!; + expect(score).toBeLessThan(full); + }); + + test("hard cooldown is authoritative: score 0 and evaluator exclusion", async () => { + const { clearCodexUpstreamHealth } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const evidence = { + sampleCount: 50, + successRate: 0.95, + cooldownUntilMs: now + 60_000, + }; + expect(healthScore(evidence, now)).toBe(0); + + const result = evaluatePolicyProfile(config(), "healthy", {}, [ + { provider: "a", model: "m1", health: { sampleCount: 50, successRate: 0.95, cooldownUntilMs: now + 60_000 } }, + { provider: "b", model: "m2", health: { sampleCount: 50, successRate: 0.95 } }, + ]); + expect(result.candidates[0]!.eligible).toBe(false); + expect(result.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cooldown")).toBe(true); + expect(result.selectedIndex).toBe(1); + }); + + test("unknown health follows the profile unknownEvidence policy", async () => { + const strict = config({ + routingProfiles: { + h: { + candidates: [{ provider: "a", model: "m1" }], + unknownEvidence: { capability: "allow", health: "exclude", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const excluded = evaluatePolicyProfile(strict, "h", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-health")).toBe(true); + expect(excluded.selectedIndex).toBeNull(); + + const penalizing = config({ + routingProfiles: { + h: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const penalized = evaluatePolicyProfile(penalizing, "h", {}, []); + // Both candidates have unknown health; penalize keeps them eligible with + // the penalized health floor folded into the score. + expect(penalized.candidates.every(candidate => candidate.eligible)).toBe(true); + expect(penalized.candidates[0]!.score!.components.health).toBe(0.3); + }); + + test("known health evidence feeds the score component and trace", async () => { + for (let index = 0; index < 30; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + const healthA = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const healthB = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + const result = evaluatePolicyProfile(config(), "healthy", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, health: healthA }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 }, health: healthB }, + ]); + const a = result.candidates[0]!; + expect(a.score!.components.health).toBeGreaterThan(0); + expect(a.score!.components.configuredPriority).toBe(1); + // Health evidence reaches the trace candidate. + expect(result.trace.candidates[0]!.health).toBeDefined(); + }); + + test("historical health moves selection between two eligible candidates", async () => { + // "a" has a bad recent streak; "b" is clean. + for (let index = 0; index < 20; index++) appendUsageEntry(row(`afail-${index}`, 503, 4000)); + for (let index = 0; index < 20; index++) { + appendUsageEntry({ + requestId: `bok-${index}`, + timestamp: Date.now() - 60_000, + provider: "b", + model: "m2", + status: 200, + durationMs: 900, + usageStatus: "reported", + }); + } + const healthA = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const healthB = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + const healthDominant = config({ + routingProfiles: { + healthy: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + optimize: { health: 0.9 }, + }, + }, + }); + const result = evaluatePolicyProfile(healthDominant, "healthy", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, health: healthA }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 }, health: healthB }, + ]); + // With equal priority weights, the healthier candidate wins. + expect(result.selectedIndex).toBe(1); + }); + + test("execution path includes health evidence and can exclude on cooldown", async () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + const route = routeModel(config(), "policy/healthy"); + expect(route.routeKind).toBe("policy"); + expect(route.routeDecision!.candidates[0]!.health).toBeDefined(); + }); +}); diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index 00c0f67d39..b4f9976ec0 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; import { getRoutingProfile } from "../src/routing/profile"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import type { OcxConfig } from "../src/types"; let testDir = ""; @@ -16,6 +17,7 @@ beforeEach(() => { }); afterEach(() => { + closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (testDir) rmSync(testDir, { recursive: true, force: true }); @@ -85,7 +87,11 @@ describe("policy execution (RI-05)", () => { expect(trace.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); expect(trace.selected.provider).toBe("a"); expect(trace.selected.model).toBe("m1"); - expect(trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + // RI-06: unknown health under the default "penalize" policy folds a + // penalized health floor into the score. + expect(trace.candidates[0]!.score).toMatchObject({ + components: { configuredPriority: 1, health: 0.3 }, + }); }); test("profile alias executes the same policy", () => { diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index 8de698b140..206eea46c3 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -234,7 +234,9 @@ describe("routing profiles (RI-04)", () => { { provider: "b", model: "m2", capability: { contextWindow: 5000 } }, ]); expect(result.selectedIndex).toBe(0); - expect(result.trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + expect(result.trace.candidates[0]!.score).toMatchObject({ + components: { configuredPriority: 1, health: 0.3 }, + }); }); test("API lists profiles and dry-runs deterministically", async () => { From 6095001f9eddee89ed0cd7ef1974cdfd2b77a971 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 05:48:46 +0200 Subject: [PATCH 06/10] fix(routing): address full-review findings and approved simplify (RI-06) --- src/router.ts | 34 +++-- src/routing/capability.ts | 17 ++- src/routing/evaluator.ts | 26 +++- src/routing/health.ts | 129 ++++++++++++++++-- src/routing/history/indexer.ts | 20 ++- src/routing/profile.ts | 12 ++ src/routing/request-evidence.ts | 23 ++-- .../management/routing-profile-routes.ts | 12 +- 8 files changed, 233 insertions(+), 40 deletions(-) diff --git a/src/router.ts b/src/router.ts index fb4272ac21..0d5fe2789f 100644 --- a/src/router.ts +++ b/src/router.ts @@ -21,6 +21,7 @@ import { import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec"; import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; +import { getEffectiveActiveCodexAccountId } from "./codex/routing"; import { buildRouteDecisionTrace, type RouteDecisionKind, @@ -30,7 +31,7 @@ import { import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; import { candidateCapabilityEvidence } from "./routing/capability"; -import { healthEvidenceForCandidate } from "./routing/health"; +import { codexPoolHealthEvidence, healthEvidenceForCandidate } from "./routing/health"; export class NoEligiblePolicyCandidateError extends Error { constructor(readonly profileId: string) { @@ -440,19 +441,32 @@ function routeModelInternal( const slash = modelId.indexOf("/"); // Policy namespace is system-reserved: an explicit `policy/` or a // configured profile alias executes the policy evaluator and routes the - // selected candidate. Only explicit requests reach this branch. - const policyId = resolvePolicyProfileId(config, modelId); - if (policyId) { - const profile = getRoutingProfile(config, policyId); - if (!profile) throw new Error(`Unknown routing profile: ${policyId}`); + // selected candidate. Only explicit requests reach this branch; concrete + // recursive targets skip policy resolution entirely (bypassCombos) so an + // alias matching a selected candidate can never recurse, and a + // `policy/` without a configured profile falls through to normal + // provider/default resolution instead of failing. + const policyId = !bypassCombos ? resolvePolicyProfileId(config, modelId) : null; + const profile = policyId ? getRoutingProfile(config, policyId) : undefined; + if (profile && policyId) { const candidateEvidence = profile.candidates.map(candidate => ({ provider: candidate.provider, model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), - health: healthEvidenceForCandidate({ - provider: candidate.provider, - model: candidate.model, - }), + health: { + ...healthEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? getEffectiveActiveCodexAccountId(config) + : undefined, + }), + // Live pool state stays authoritative for `openai` targets even when + // no account reference exists in the candidate evidence. + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? (codexPoolHealthEvidence(config) ?? {}) + : {}), + }, })); const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); if (evaluation.selectedIndex === null) { diff --git a/src/routing/capability.ts b/src/routing/capability.ts index 378574886c..dad65dd128 100644 --- a/src/routing/capability.ts +++ b/src/routing/capability.ts @@ -61,6 +61,16 @@ function isPrivateHostname(hostname: string): boolean { || /^172\.(1[6-9]|2\d|3[01])\./.test(hostname); } +/** Adapters whose upstream protocol supports function/tool calling. */ +const TOOL_CAPABLE_ADAPTERS = new Set([ + "openai-chat", + "openai-responses", + "anthropic", + "cursor", + "google", + "azure-openai", +]); + function localRemoteEvidence(baseUrl: string | undefined): Pick { if (typeof baseUrl !== "string" || baseUrl.length === 0) return {}; try { @@ -103,8 +113,13 @@ export function candidateCapabilityEvidence( : undefined; const capabilities = catalogRow?.capabilities ?? []; + // The catalog capability is the per-model authority. Without a catalog row, + // the adapter protocol itself is the signal: `openai-chat` and friends run + // single tool calls even when the parallel-call opt-in is unset or false. const tools = capabilities.includes("tools") - || (isNative ? true : provider?.parallelToolCalls === true) + || (isNative ? true : false) + || (catalogRow === undefined && provider !== undefined && TOOL_CAPABLE_ADAPTERS.has(provider.adapter)) + || provider?.parallelToolCalls === true || undefined; const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId] diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 0237650eb2..7eafd276e8 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -166,6 +166,31 @@ function requestRequirements( const image = booleanRequirement("request-image-input", true, capability?.image); if (image) requirements.push(image); } + if (requestEvidence.contextWindow !== undefined) { + const actual = capability?.contextWindow; + if (typeof actual === "number") { + requirements.push({ + id: "request-context-window", + expected: requestEvidence.contextWindow, + actual, + outcome: actual >= requestEvidence.contextWindow ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ + id: "request-context-window", + expected: requestEvidence.contextWindow, + outcome: "unknown", + }); + } + } + if (requestEvidence.structuredOutputRequired === true) { + const structured = booleanRequirement("request-structured-output", true, capability?.structuredOutput); + if (structured) requirements.push(structured); + } + if (requestEvidence.encryptedCodexTask === true) { + const encrypted = booleanRequirement("request-encrypted-codex-task", true, capability?.encryptedCodexTasks); + if (encrypted) requirements.push(encrypted); + } return requirements; } @@ -257,7 +282,6 @@ export function evaluatePolicyProfile( ...(evidence.health ? { health: evidence.health } : {}), ...(evidence.quota ? { quota: evidence.quota } : {}), ...(evidence.cost ? { cost: evidence.cost } : {}), - ...(evidence.health ? { health: evidence.health } : {}), score, }; candidates.push(evaluated); diff --git a/src/routing/health.ts b/src/routing/health.ts index 0656efb2b9..b90f4450e5 100644 --- a/src/routing/health.ts +++ b/src/routing/health.ts @@ -21,7 +21,9 @@ import { openRequestHistoryIndexSync, requestHistoryDb } from "./history/indexer import { getCodexAccountCooldownUntil, getCodexAccountSoftAvoidUntil, + getEffectiveActiveCodexAccountId, isCodexAccountInCooldown, + listLiveCodexAccountIds, } from "../codex/routing"; import type { RouteHealthEvidence } from "./trace"; @@ -64,6 +66,83 @@ interface HealthSample { timestamp: number; } +interface HealthRow extends HealthSample { + attemptCount?: number; + rowJson?: string | null; +} + +/** + * Per-attempt samples for a candidate from a row's persisted entry. + * Combo/failover requests store each upstream try in `entry.attempts` while + * the top-level row records the final outcome; a provider/model that failed + * as a non-final attempt must still contribute its own health samples. + */ +function attemptSamplesFor( + row: Pick, + provider: string, + model: string, +): HealthSample[] { + if (!row.rowJson || (row.attemptCount ?? 1) <= 1) return []; + try { + const parsed = JSON.parse(row.rowJson) as { attempts?: unknown }; + if (!Array.isArray(parsed.attempts)) return []; + const samples: HealthSample[] = []; + for (const attempt of parsed.attempts) { + if (!attempt || typeof attempt !== "object" || Array.isArray(attempt)) continue; + const record = attempt as Record; + if (record.provider !== provider || record.model !== model) continue; + if (typeof record.status !== "number" || typeof record.durationMs !== "number") continue; + samples.push({ + status: record.status, + closeReason: null, + terminalStatus: null, + durationMs: record.durationMs, + timestamp: row.timestamp, + }); + } + return samples; + } catch { + return []; + } +} + +/** + * Live Codex pool account state for an `openai` policy candidate. + * + * When a deterministic active account exists (manual selection or + * `config.activeCodexAccountId`) its cooldown/soft-avoid state is + * authoritative. Otherwise the target is conservatively cooled only when + * every live pool account is cooling or soft-avoided; account selection + * inside `src/codex/routing.ts` stays authoritative when any account is + * usable, so policy scoring never invents account choices. + */ +export function codexPoolHealthEvidence( + config: Parameters[0], + now = Date.now(), +): Pick | undefined { + const activeId = getEffectiveActiveCodexAccountId(config); + if (activeId) { + const until = getCodexAccountCooldownUntil(activeId, now); + if (until !== null) return { cooldownUntilMs: until }; + const softUntil = getCodexAccountSoftAvoidUntil(activeId, now); + if (softUntil !== null && softUntil > now) return { softAvoidUntilMs: softUntil }; + return undefined; + } + const live = [...listLiveCodexAccountIds(config)]; + if (live.length === 0) return undefined; + const cooldowns: number[] = []; + const softAvoids: number[] = []; + for (const accountId of live) { + const until = getCodexAccountCooldownUntil(accountId, now); + if (until !== null) cooldowns.push(until); + const softUntil = getCodexAccountSoftAvoidUntil(accountId, now); + if (softUntil !== null && softUntil > now) softAvoids.push(softUntil); + } + if (cooldowns.length === live.length) return { cooldownUntilMs: Math.max(...cooldowns) }; + if (softAvoids.length === live.length) return { softAvoidUntilMs: Math.max(...softAvoids) }; + return undefined; +} + function classifySample(sample: HealthSample): "success" | "failure" | "neutral" { if (sample.closeReason === "client_cancel" || sample.status === 499) return "neutral"; // Invalid requests and policy refusals must not poison target health. @@ -115,10 +194,33 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea } const rows = handle.query( `SELECT status, close_reason AS closeReason, terminal_status AS terminalStatus, - duration_ms AS durationMs, timestamp + duration_ms AS durationMs, timestamp, + attempt_count AS attemptCount, row_json AS rowJson FROM requests WHERE ${where.join(" AND ")} ORDER BY timestamp DESC LIMIT ?`, - ).all(...values, HEALTH_MAX_SAMPLES) as HealthSample[]; + ).all(...values, HEALTH_MAX_SAMPLES) as HealthRow[]; + // Rows whose top-level target differs from this candidate may still carry + // candidate attempts (combo/failover): expand those too. + const attemptRows = handle.query( + `SELECT timestamp, attempt_count AS attemptCount, row_json AS rowJson + FROM requests WHERE timestamp >= ? AND attempt_count > 1 + AND NOT (provider = ? AND model = ?) + ORDER BY timestamp DESC LIMIT ?`, + ).all(now - HEALTH_WINDOW_MS, input.provider, input.model, HEALTH_MAX_SAMPLES) as Array< + Pick + >; + + const samples: HealthSample[] = []; + for (const row of rows) { + const attemptSamples = attemptSamplesFor(row, input.provider, input.model); + samples.push(...(attemptSamples.length > 0 ? attemptSamples : [row])); + } + for (const row of attemptRows) { + samples.push(...attemptSamplesFor(row, input.provider, input.model)); + } + // Newest first for the consecutive-failure walk; attempt samples inherit + // their row's timestamp. + samples.sort((a, b) => b.timestamp - a.timestamp); let successes = 0; let failures = 0; @@ -126,11 +228,9 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea let weightedSuccess = 0; let weightedTotal = 0; const latencies: number[] = []; - let consecutiveFailures = 0; - for (let index = 0; index < rows.length; index++) { - const row = rows[index]!; - const kind = classifySample(row); - const weight = decayWeight(row.timestamp, now); + for (const sample of samples) { + const kind = classifySample(sample); + const weight = decayWeight(sample.timestamp, now); if (kind === "neutral") continue; if (kind === "success") { successes += 1; @@ -140,18 +240,17 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea failures += 1; weightedTotal += weight; } - if (row.terminalStatus === "incomplete") incompleteStreams += 1; - latencies.push(row.durationMs); + if (sample.terminalStatus === "incomplete") incompleteStreams += 1; + latencies.push(sample.durationMs); } // Consecutive failures: walk newest -> oldest until a success. - let consecutive = 0; - for (const row of rows) { - const kind = classifySample(row); + let consecutiveFailures = 0; + for (const sample of samples) { + const kind = classifySample(sample); if (kind === "neutral") continue; - if (kind === "failure") consecutive += 1; + if (kind === "failure") consecutiveFailures += 1; else break; } - consecutiveFailures = consecutive; const sampleCount = successes + failures; if (sampleCount > 0) { @@ -162,7 +261,7 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea latencies.sort((a, b) => a - b); const p50 = median(latencies); if (p50 !== undefined) evidence.recentLatencyMs = p50; - evidence.recencyWeight = decayWeight(rows[0]!.timestamp, now); + evidence.recencyWeight = decayWeight(samples[0]!.timestamp, now); } } catch { /* index unreadable: evidence stays unknown */ diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index bd0e56489f..f8dddfc804 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -128,8 +128,14 @@ function sourceIdentity(): UsageLogRevision | null { function sourceIdentityMatches(dbHandle: Database, revision: UsageLogRevision | null): boolean { const stored = readIndexedMeta(dbHandle); if (revision === null) return stored.sourceSize === 0; - return stored.sourceSize === Number(revision.size) - && stored.sourceMtimeMs === Number(revision.mtimeMs); + const storedDev = Number(metaValue(dbHandle, HISTORY_META_KEYS.sourceDev)); + const storedIno = Number(metaValue(dbHandle, HISTORY_META_KEYS.sourceIno)); + // Stable file identity (dev/ino) is authoritative; size/mtime growth is a + // tail the incremental offset logic ingests, never a rebuild trigger. + // Without this, every appended usage row changed size/mtime and forced a + // synchronous full rebuild on the next routing-time health read. + return storedDev === Number(revision.dev) + && storedIno === Number(revision.ino); } /** Extract the `requests` row columns from a canonical persisted entry. */ @@ -203,7 +209,10 @@ function parsedEntryFromLine(line: string): PersistedUsageEntry | null { if (parsed && typeof parsed === "object" && typeof parsed.requestId === "string" && typeof parsed.timestamp === "number" - && typeof parsed.provider === "string") { + && typeof parsed.provider === "string" + && typeof parsed.model === "string" + && typeof parsed.status === "number" + && typeof parsed.durationMs === "number") { return parsed; } } catch { @@ -415,7 +424,10 @@ function refreshLockedSync(): RequestHistoryIndexMeta { return metaFor(handle); } if (tailNextOffset < Number(revision.size)) { - ingestSourceTail(handle, revision.path, tailNextOffset); + const inserted = ingestSourceTail(handle, revision.path, tailNextOffset); + // A clean tail ingest proves the index is healthy: clear any earlier + // rebuild marker so status readers can distinguish rebuilds from tails. + if (inserted > 0) setMeta(handle, HISTORY_META_KEYS.lastError, ""); } return metaFor(handle); } diff --git a/src/routing/profile.ts b/src/routing/profile.ts index c0a724aa67..9fcb5214c8 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -153,6 +153,18 @@ function aliasIssues( if (hasOwnProvider(config.providers, alias)) { issues.push({ path: ["alias"], message: `alias "${alias}" collides with configured provider name "${alias}"` }); } + // A slashed alias whose first segment is a configured provider shadows that + // provider's routes (e.g. `a/m1` resolves as the alias before the provider + // namespace). Reject those aliases to preserve existing routing. + if (alias.includes("/")) { + const firstSegment = alias.split("/")[0]!; + if (hasOwnProvider(config.providers, firstSegment)) { + issues.push({ + path: ["alias"], + message: `alias "${alias}" collides with configured provider namespace "${firstSegment}"`, + }); + } + } if (resolveComboId({ combos: config.combos }, alias)) { issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured combo selector` }); } diff --git a/src/routing/request-evidence.ts b/src/routing/request-evidence.ts index 66fe34fc41..39c666b5cd 100644 --- a/src/routing/request-evidence.ts +++ b/src/routing/request-evidence.ts @@ -9,23 +9,30 @@ import type { PolicyRequestEvidence } from "./evaluator"; +function partContainsImage(part: unknown): boolean { + if (!part || typeof part !== "object" || Array.isArray(part)) return false; + const record = part as Record; + if (record.type === "image" || record.type === "input_image") return true; + if (record.image_url !== undefined || record.image !== undefined) return true; + // Responses/Chat/Claude nest image parts under message `content` arrays + // (e.g. `{ type: "message", content: [{ type: "input_image", ... }] }`); + // walk nested arrays so vision requests are not missed. + if (Array.isArray(record.content)) return record.content.some(partContainsImage); + return false; +} + function inputContainsImage(input: unknown): boolean { if (typeof input === "string") return false; if (!Array.isArray(input)) return false; - return input.some(part => { - if (!part || typeof part !== "object" || Array.isArray(part)) return false; - const record = part as Record; - if (record.type === "image" || record.type === "input_image") return true; - if (record.image_url !== undefined || record.image !== undefined) return true; - return false; - }); + return input.some(partContainsImage); } export function evidenceFromBody(body: unknown): PolicyRequestEvidence { if (!body || typeof body !== "object" || Array.isArray(body)) return {}; const record = body as Record; const tools = Array.isArray(record.tools) && record.tools.length > 0; - const image = inputContainsImage(record.input); + // `input` (Responses) or `messages` (Chat/Claude) both carry image parts. + const image = inputContainsImage(record.input) || inputContainsImage(record.messages); return { ...(tools ? { toolsRequired: true } : {}), ...(image ? { imageInputRequired: true } : {}), diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index a9d641d6da..398f67a860 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -8,6 +8,8 @@ import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from "../../routing/profile"; import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; +import { candidateCapabilityEvidence } from "../../routing/capability"; +import { healthEvidenceForCandidate } from "../../routing/health"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; @@ -98,7 +100,15 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config); } const candidateEvidence = body.candidates === undefined - ? [] + // Match execution: fill the same candidate evidence the router would + // assemble, so dry-run reports the same eligibility as real routing + // instead of treating every capability as unknown. + ? getRoutingProfile(config, profile)!.candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + health: healthEvidenceForCandidate({ provider: candidate.provider, model: candidate.model }), + })) : parseCandidateEvidence(body.candidates); if (candidateEvidence === null) { return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config); From a9c6f8e875afceed1533cd6ef2342204187b60d8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 05:48:54 +0200 Subject: [PATCH 07/10] test(routing): cover review-round fixes for health, policy, and indexer (RI-06) --- .../001_pr_stack_status.md | 31 ++++++++ tests/health-scoring.test.ts | 47 +++++++++++++ tests/policy-execution.test.ts | 70 +++++++++++++++++-- tests/request-history-index.test.ts | 17 +++++ tests/routing-profile.test.ts | 41 +++++++++++ 5 files changed, 200 insertions(+), 6 deletions(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index b15e3db8f9..8babdcc50e 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -207,6 +207,37 @@ other; closing one is a maintainer decision and neither is stale. - `bun run privacy:scan`: passed - Remaining Low findings: none +### RI-06 review round (full-review #1013 + simplify) + +- Base SHA: `56f17f45c` (RI-05 head); PR head `909ce21d4` before fixes. +- Simplify (approved): duplicate `health` spread removed in the evaluator; + dead `consecutiveFailures` initializer removed in `health.ts`. +- Bot-thread + own findings fixed in this PR: + 1. indexer: appends no longer trigger a full synchronous rebuild - identity + is dev/ino, growth is a tail (`sourceIdentityMatches`); + 2. router: policy candidates now carry live Codex pool account + cooldown/soft-avoid evidence for `openai` targets + (`codexPoolHealthEvidence` + active account); + 3. health: combo/failover `attempts[]` expand into per-target samples; + 4. request evidence: nested message `content` arrays (Responses/Chat/Claude) + are walked for images; + 5. evaluator: request-side `contextWindow`, `structuredOutputRequired`, + `encryptedCodexTask` are enforced like tools/image; + 6. dry-run API: omitted `candidates` now populate the same evidence as + execution (capability + health) instead of evaluating against none; + 7. alias validation rejects slashed aliases whose first segment is a + configured provider; + 8. `policy/` without a configured profile falls through to normal + resolution instead of throwing; + 9. capability: adapter-level tool support inferred for tool-capable + adapters when no catalog row exists. +- Verification: `tsc --noEmit` 0 errors; focused suites green (74/74 in the + routing set, 330/336 incl. config - the 6 failures are Windows symlink + `EPERM` environment failures, present on the baseline); `privacy:scan` + passed. +- Base sync deferred: waiting for RI-05 (#1012) to merge before updating + these branches from `dev` (PR heads currently conflict with `dev`). + ## Baseline note The full-suite baseline on this Windows machine did not complete within the diff --git a/tests/health-scoring.test.ts b/tests/health-scoring.test.ts index bcc4c76ab0..7d4179c8ac 100644 --- a/tests/health-scoring.test.ts +++ b/tests/health-scoring.test.ts @@ -219,4 +219,51 @@ describe("health-aware scoring (RI-06)", () => { expect(route.routeKind).toBe("policy"); expect(route.routeDecision!.candidates[0]!.health).toBeDefined(); }); + + test("combo attempt failures contribute samples to the failed target", async () => { + for (let index = 0; index < 10; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + // Combo request: a/m1 failed as the non-final attempt, b/m2 succeeded. + appendUsageEntry({ + ...row("combo-1", 200, 1500, { provider: "b", model: "m2" }), + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 4000, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + { ordinal: 2, provider: "b", model: "m2", adapter: "openai-chat", status: 200, durationMs: 1500, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + }); + const failedTarget = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(failedTarget.sampleCount).toBe(11); + expect(failedTarget.failures).toBe(1); + const finalTarget = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + expect(finalTarget.sampleCount).toBe(1); + expect(finalTarget.failures).toBeUndefined(); + expect(finalTarget.successRate).toBe(1); + }); + + test("execution path applies live codex account cooldown to openai candidates", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const cfg = config({ + providers: { + ...config().providers, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + mixed: { + candidates: [ + { provider: "openai", model: "gpt-5.6" }, + { provider: "b", model: "m2" }, + ], + }, + }, + }); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { retryAfter: "3600", now }); + const route = routeModel(cfg, "policy/mixed"); + expect(route.routeDecision!.candidates[0]!.health?.cooldownUntilMs).toBeDefined(); + expect(route.routeDecision!.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cooldown")).toBe(true); + expect(route.providerName).toBe("b"); + expect(route.modelId).toBe("m2"); + }); }); diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index b4f9976ec0..baddedeea2 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; import { getRoutingProfile } from "../src/routing/profile"; +import { evidenceFromBody } from "../src/routing/request-evidence"; import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import type { OcxConfig } from "../src/types"; @@ -45,6 +46,16 @@ function baseConfig(overrides: Partial = {}): OcxConfig { modelContextWindows: { m2: 64_000 }, modelInputModalities: { m2: ["text"] }, }, + // Non-tool-capable adapter: keeps the "tools unknown" scenario testable + // now that `openai-chat` infers tool support from the adapter. + c: { + adapter: "bare", + baseUrl: "https://c.example/v1", + apiKey: "kc", + models: ["m3"], + modelContextWindows: { m3: 128_000 }, + modelInputModalities: { m3: ["text"] }, + }, openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, }, combos: { @@ -122,11 +133,11 @@ describe("policy execution (RI-05)", () => { }); test("unknown capability follows the profile unknownEvidence (exclude default)", () => { - // Provider "b" has no parallelToolCalls and no catalog row: tools unknown. + // Provider "c" uses a non-tool-capable adapter and no catalog row: tools unknown. const config = baseConfig({ routingProfiles: { toolsOnly: { - candidates: [{ provider: "b", model: "m2" }], + candidates: [{ provider: "c", model: "m3" }], require: { tools: true }, }, }, @@ -136,15 +147,29 @@ describe("policy execution (RI-05)", () => { const permissive = baseConfig({ routingProfiles: { toolsOnly: { - candidates: [{ provider: "b", model: "m2" }], + candidates: [{ provider: "c", model: "m3" }], require: { tools: true }, unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, }, }, }); const route = routeModel(permissive, "policy/toolsOnly"); + expect(route.providerName).toBe("c"); + expect(route.modelId).toBe("m3"); + }); + + test("openai-chat without the parallel-call opt-in still infers tool support", () => { + // Provider "b" is `openai-chat` with no `parallelToolCalls` and no catalog + // row: the adapter protocol itself is the tool-capability signal. + const config = baseConfig({ + routingProfiles: { + tools: { candidates: [{ provider: "b", model: "m2" }], require: { tools: true } }, + }, + }); + const route = routeModel(config, "policy/tools"); expect(route.providerName).toBe("b"); expect(route.modelId).toBe("m2"); + expect(route.routeDecision!.candidates[0]!.capability?.tools).toBe(true); }); test("request evidence constrains candidates: image input excludes non-image models", () => { @@ -163,14 +188,47 @@ describe("policy execution (RI-05)", () => { test("request tools requirement is enforced when provably needed", () => { const config = baseConfig({ routingProfiles: { - tools: { candidates: [{ provider: "b", model: "m2" }] }, + tools: { candidates: [{ provider: "c", model: "m3" }] }, }, }); - expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "b", modelId: "m2" }); - // b's tools support is unknown -> request requiring tools excludes it. + expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "c", modelId: "m3" }); + // c's tools support is unknown -> request requiring tools excludes it. expect(() => routeModel(config, "policy/tools", { toolsRequired: true })).toThrow(NoEligiblePolicyCandidateError); }); + test("request evidence walks nested message content for images", () => { + const responses = evidenceFromBody({ + input: [{ + type: "message", + role: "user", + content: [{ type: "input_image", image_url: "https://example.test/x.png" }], + }], + }); + expect(responses.imageInputRequired).toBe(true); + + const chat = evidenceFromBody({ + messages: [{ + role: "user", + content: [{ type: "image_url", image_url: { url: "https://example.test/y.png" } }], + }], + }); + expect(chat.imageInputRequired).toBe(true); + + const plain = evidenceFromBody({ + input: [{ type: "message", role: "user", content: "just text" }], + }); + expect(plain.imageInputRequired).toBeUndefined(); + }); + + test("unresolved policy/ falls through to normal resolution", () => { + const config = baseConfig(); + // No profile named "nope": the reserved-looking id must not throw and not + // shadow provider/default resolution. + const route = routeModel(config, "policy/nope"); + expect(route.routeKind).toBe("default-provider"); + expect(route.providerName).toBe("a"); + }); + test("policy selection is deterministic across calls", () => { const config = baseConfig(); const first = routeModel(config, "policy/fast"); diff --git a/tests/request-history-index.test.ts b/tests/request-history-index.test.ts index 466ef89b5f..45de8105da 100644 --- a/tests/request-history-index.test.ts +++ b/tests/request-history-index.test.ts @@ -105,6 +105,23 @@ describe("request-history index (RI-02)", () => { expect(after.meta.indexedRows).toBe(6); }); + test("appended rows are ingested as a tail, never a full rebuild", async () => { + for (const row of seedRows(5)) appendUsageEntry(row); + const first = await queryRequestHistory({}, undefined, 10); + expect(first.meta.indexedRows).toBe(5); + // The initial build reports a rebuild; a subsequent append must be + // ingested from the indexed offset without rebuilding (routing-time + // health reads depend on this not re-parsing the whole ledger). + appendUsageEntry(entry("late-1", 7000)); + const second = await queryRequestHistory({}, undefined, 10); + expect(second.meta.indexedRows).toBe(6); + expect(second.meta.lastError).toBe(""); + appendUsageEntry(entry("late-2", 8000)); + const third = await queryRequestHistory({}, undefined, 10); + expect(third.meta.indexedRows).toBe(7); + expect(third.meta.lastError).toBe(""); + }); + test("large history indexes fully and paginates without duplicates or misses", async () => { const rows = seedRows(1500, 10_000); for (const row of rows) appendUsageEntry(row); diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index 206eea46c3..c37ce7012e 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { validateConfigCandidate } from "../src/config"; import { handleManagementAPI } from "../src/server/management-api"; import { ManagementRequest } from "./helpers/management-auth"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import { getRoutingProfile, listRoutingProfileIds, @@ -27,6 +28,7 @@ beforeEach(() => { }); afterEach(() => { + closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (testDir) rmSync(testDir, { recursive: true, force: true }); @@ -117,6 +119,12 @@ describe("routing profiles (RI-04)", () => { }, config); expect(providerCollision.some(issue => issue.message.includes("provider name"))).toBe(true); + const providerNamespaceCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "a/m1", + }, config); + expect(providerNamespaceCollision.some(issue => issue.message.includes("provider namespace"))).toBe(true); + const comboCollision = routingProfileIssues("p", { candidates: [{ provider: "a", model: "m1" }], alias: "combo/free", @@ -286,4 +294,37 @@ describe("routing profiles (RI-04)", () => { const badResponse = await handleManagementAPI(badReq, new URL(badReq.url), config, { refreshCodexCatalog: async () => {} }); expect(badResponse!.status).toBe(400); }); + + test("API dry-run without explicit candidates fills the same evidence as execution", async () => { + const config = baseConfig({ + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"], modelContextWindows: { m1: 200_000 }, parallelToolCalls: true }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"], modelContextWindows: { m2: 64_000 } }, + }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + }, + }, + }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { selectedIndex?: number | null; candidates?: Array<{ provider?: string; eligible?: boolean }> }; + // a: 200k context + openai-chat tools => eligible; b: 64k below the hard + // minimum => excluded, exactly like real routing would report. + expect(body.selectedIndex).toBe(0); + expect(body.candidates?.[0]).toMatchObject({ provider: "a", eligible: true }); + expect(body.candidates?.[1]).toMatchObject({ provider: "b", eligible: false }); + }); }); From 9f997f88bfc640c6e3189f040bc28e7fc700d1c9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 05:55:50 +0200 Subject: [PATCH 08/10] test(routing): make combo attempt health sample ordering deterministic --- tests/health-scoring.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/health-scoring.test.ts b/tests/health-scoring.test.ts index 7d4179c8ac..b14f8175ec 100644 --- a/tests/health-scoring.test.ts +++ b/tests/health-scoring.test.ts @@ -224,7 +224,7 @@ describe("health-aware scoring (RI-06)", () => { for (let index = 0; index < 10; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); // Combo request: a/m1 failed as the non-final attempt, b/m2 succeeded. appendUsageEntry({ - ...row("combo-1", 200, 1500, { provider: "b", model: "m2" }), + ...row("combo-1", 200, 1500, { provider: "b", model: "m2", timestamp: Date.now() - 1_000 }), attempts: [ { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 4000, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, { ordinal: 2, provider: "b", model: "m2", adapter: "openai-chat", status: 200, durationMs: 1500, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, From af692bb7ae42ac5649abf270979aefb5d5367939 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:02:15 +0200 Subject: [PATCH 09/10] fix(routing): address CodeRabbit review round (RI-06) - thread one clock read through policy health evaluation (router + evaluator) - blend unknown health under 'allow' at a neutral 0.5 instead of outranking measured health - mixed pool cooldown/soft-avoid states degrade to soft-avoid - TTL-cache historical health evidence; live cooldown stays uncached - LIKE-prefilter attempt-row scan so LIMIT is not starved by non-matching rows - add kiro/mimo-free/azure to tool-capable adapters - remove duplicate provider-namespace alias check (dev already had it) - whitelist + bound candidate evidence in trace builder - dry-run mirrors the router's health assembly via shared helper - regression tests for all of the above --- src/router.ts | 23 +-- src/routing/capability.ts | 21 ++- src/routing/evaluator.ts | 11 +- src/routing/health.ts | 150 +++++++++++++++--- src/routing/profile.ts | 9 -- src/routing/trace.ts | 16 +- .../management/routing-profile-routes.ts | 9 +- tests/health-scoring.test.ts | 59 ++++++- tests/policy-execution.test.ts | 16 ++ tests/route-decision-trace.test.ts | 28 ++++ tests/routing-profile.test.ts | 37 ++++- 11 files changed, 311 insertions(+), 68 deletions(-) diff --git a/src/router.ts b/src/router.ts index 5b821e5efa..61a25b845e 100644 --- a/src/router.ts +++ b/src/router.ts @@ -22,7 +22,6 @@ import { import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec"; import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; -import { getEffectiveActiveCodexAccountId } from "./codex/routing"; import { buildRouteDecisionTrace, type RouteDecisionKind, @@ -32,7 +31,7 @@ import { import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; import { candidateCapabilityEvidence } from "./routing/capability"; -import { codexPoolHealthEvidence, healthEvidenceForCandidate } from "./routing/health"; +import { policyCandidateHealthEvidence } from "./routing/health"; export class NoEligiblePolicyCandidateError extends Error { /** Evaluation trace (with per-candidate exclusions) when nothing qualified. */ @@ -499,26 +498,16 @@ function routeModelInternal( const policyId = !bypassCombos ? resolvePolicyProfileId(config, modelId) : null; const profile = policyId ? getRoutingProfile(config, policyId) : undefined; if (profile && policyId) { + // One clock read per decision keeps candidate evidence, exclusions, and + // scores mutually consistent and reproducible. + const now = Date.now(); const candidateEvidence = profile.candidates.map(candidate => ({ provider: candidate.provider, model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), - health: { - ...healthEvidenceForCandidate({ - provider: candidate.provider, - model: candidate.model, - codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID - ? getEffectiveActiveCodexAccountId(config) - : undefined, - }), - // Live pool state stays authoritative for `openai` targets even when - // no account reference exists in the candidate evidence. - ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID - ? (codexPoolHealthEvidence(config) ?? {}) - : {}), - }, + health: policyCandidateHealthEvidence(config, candidate, now), })); - const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); + const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence, now); if (evaluation.selectedIndex === null) { throw new NoEligiblePolicyCandidateError(policyId, evaluation.trace); } diff --git a/src/routing/capability.ts b/src/routing/capability.ts index d05e1ba3b0..b628e55df8 100644 --- a/src/routing/capability.ts +++ b/src/routing/capability.ts @@ -96,7 +96,11 @@ function classifyHostname(hostname: string): "local" | "private" | null { return null; } -/** Adapters whose upstream protocol supports function/tool calling. */ +/** + * Adapters whose upstream protocol supports function/tool calling. Mirrors + * the adapter ids the resolver accepts (including the `azure` alias for + * `azure-openai`); `kiro` and `mimo-free` send/delegate tool calls. + */ const TOOL_CAPABLE_ADAPTERS = new Set([ "openai-chat", "openai-responses", @@ -104,6 +108,9 @@ const TOOL_CAPABLE_ADAPTERS = new Set([ "cursor", "google", "azure-openai", + "azure", + "kiro", + "mimo-free", ]); function localRemoteEvidence(baseUrl: string | undefined): Pick { @@ -154,12 +161,14 @@ export function candidateCapabilityEvidence( : undefined; const capabilities = catalogRow?.capabilities ?? []; - // The catalog capability is the per-model authority. Without a catalog row, - // the adapter protocol itself is the signal: `openai-chat` and friends run - // single tool calls even when the parallel-call opt-in is unset or false. - // `parallelToolCalls` stays a positive provider-level override. + // The catalog `capabilities` list is a positive per-model signal; a row + // without "tools" is treated as unknown, never as a negative. Without a + // catalog row the adapter protocol itself is the signal: tool-capable + // adapters run single tool calls even when the parallel-call opt-in is + // unset or false. `parallelToolCalls` stays a positive provider-level + // override. const tools = capabilities.includes("tools") - || (isNative ? true : false) + || isNative || (catalogRow === undefined && provider !== undefined && TOOL_CAPABLE_ADAPTERS.has(provider.adapter)) || provider?.parallelToolCalls === true || undefined; diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 9f5151e492..363f97d06e 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -25,6 +25,8 @@ import { healthScore } from "./health"; /** Unknown health under "penalize": a low-but-not-zero deterministic floor. */ export const HEALTH_UNKNOWN_PENALTY_SCORE = 0.3; +/** Unknown health under "allow": neutral midpoint of the [0,1] health scale. */ +export const HEALTH_UNKNOWN_NEUTRAL_SCORE = 0.5; export interface PolicyRequestEvidence { /** Required context window for this request (tokens). */ @@ -242,6 +244,7 @@ export function evaluatePolicyProfile( profileId: string, requestEvidence: PolicyRequestEvidence, candidateEvidence: PolicyCandidateEvidence[], + now = Date.now(), ): PolicyEvaluationResult { const profile = getRoutingProfile(config, profileId); if (!profile) throw new Error(`Unknown routing profile: ${profileId}`); @@ -289,8 +292,8 @@ export function evaluatePolicyProfile( // excludes; unknown health follows the profile's unknownEvidence policy; // historical health never overrides explicit ineligibility. const health = evidence.health; - let healthValue = health ? healthScore(health) : null; - if (health?.cooldownUntilMs !== undefined && health.cooldownUntilMs > Date.now()) { + let healthValue = health ? healthScore(health, now) : null; + if (health?.cooldownUntilMs !== undefined && health.cooldownUntilMs > now) { exclusions.push({ code: "cooldown" }); eligible = false; } else if (healthValue === null && profile.unknownEvidence.health === "exclude") { @@ -298,6 +301,10 @@ export function evaluatePolicyProfile( eligible = false; } else if (healthValue === null && profile.unknownEvidence.health === "penalize") { healthValue = HEALTH_UNKNOWN_PENALTY_SCORE; + } else if (healthValue === null && profile.unknownEvidence.health === "allow") { + // Neutral midpoint: blending keeps an unknown candidate from outranking + // a measured one with the same configured priority. + healthValue = HEALTH_UNKNOWN_NEUTRAL_SCORE; } const priorityScore = configuredPriorityScore(index, profile.candidates.length); diff --git a/src/routing/health.ts b/src/routing/health.ts index b90f4450e5..f1bee547e7 100644 --- a/src/routing/health.ts +++ b/src/routing/health.ts @@ -18,6 +18,7 @@ import type { OcxConfig } from "../types"; import { openRequestHistoryIndexSync, requestHistoryDb } from "./history/indexer"; +import { OPENAI_CODEX_PROVIDER_ID } from "../providers/openai-tiers"; import { getCodexAccountCooldownUntil, getCodexAccountSoftAvoidUntil, @@ -49,6 +50,32 @@ export const HEALTH_SCORE_CONSTANTS = { export const HEALTH_WINDOW_MS = 14 * 86_400_000; export const HEALTH_MAX_SAMPLES = 100; +/** + * Historical health evidence is cached briefly across candidates within one + * routing decision (and rapid successive decisions). Live cooldown/soft-avoid + * state is always read fresh - it is never cached. The TTL bounds how stale + * the history index read may be; 1.5s keeps routing deterministic within a + * decision while bounding per-candidate SQLite work. + */ +const HEALTH_HISTORY_CACHE_TTL_MS = 1_500; +const HEALTH_HISTORY_CACHE_MAX_ENTRIES = 64; + +type HistoricalHealthEvidence = Pick< + RouteHealthEvidence, + "sampleCount" | "successRate" | "failures" | "incompleteStreamRate" | "recentLatencyMs" | "recencyWeight" +>; + +const healthHistoryCache = new Map(); + +/** Test seam: routing tests append fresh rows and must not see cached history. */ +export function clearHealthHistoryCacheForTests(): void { + healthHistoryCache.clear(); +} + +function healthHistoryCacheKey(input: Pick): string { + return `${input.provider}\u0000${input.model}\u0000${input.accountRef ?? ""}`; +} + export interface HealthEvidenceInput { provider: string; model: string; @@ -139,10 +166,43 @@ export function codexPoolHealthEvidence( if (softUntil !== null && softUntil > now) softAvoids.push(softUntil); } if (cooldowns.length === live.length) return { cooldownUntilMs: Math.max(...cooldowns) }; - if (softAvoids.length === live.length) return { softAvoidUntilMs: Math.max(...softAvoids) }; + // Every account is unavailable, but not uniformly hard-cooled (e.g. some in + // cooldown, some soft-avoided): degrade to soft-avoid with the latest expiry + // so scoring still penalizes the pool. + if (cooldowns.length + softAvoids.length >= live.length && softAvoids.length > 0) { + return { softAvoidUntilMs: Math.max(...softAvoids, ...cooldowns) }; + } return undefined; } +/** + * Candidate health evidence assembled exactly like the router's policy path: + * historical evidence plus authoritative live Codex pool state for `openai` + * targets. Shared by the router and the dry-run management route so the two + * surfaces cannot drift apart. + */ +export function policyCandidateHealthEvidence( + config: Parameters[0], + candidate: { provider: string; model: string }, + now = Date.now(), +): RouteHealthEvidence { + return { + ...healthEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? getEffectiveActiveCodexAccountId(config) + : undefined, + now, + }), + // Live pool state stays authoritative for `openai` targets even when no + // account reference exists in the candidate evidence. + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? (codexPoolHealthEvidence(config, now) ?? {}) + : {}), + }; +} + function classifySample(sample: HealthSample): "success" | "failure" | "neutral" { if (sample.closeReason === "client_cancel" || sample.status === 499) return "neutral"; // Invalid requests and policy refusals must not poison target health. @@ -168,21 +228,10 @@ function median(sorted: number[]): number | undefined { * Historical health evidence from the derived index (synchronous: called at * routing time). Never throws; an unopened/unreadable index yields unknown. */ -export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHealthEvidence { - const now = input.now ?? Date.now(); - const evidence: RouteHealthEvidence = {}; - - // Live authoritative state: hard cooldown and soft-avoid for Codex pool - // accounts. Cooldown stays authoritative over any historical score. - if (input.codexAccountId && input.provider === "openai") { - if (isCodexAccountInCooldown(input.codexAccountId, now)) { - const until = getCodexAccountCooldownUntil(input.codexAccountId, now); - if (until !== null) evidence.cooldownUntilMs = until; - } - const softAvoidUntil = getCodexAccountSoftAvoidUntil(input.codexAccountId, now); - if (softAvoidUntil !== null && softAvoidUntil > now) evidence.softAvoidUntilMs = softAvoidUntil; - } - +function computeHistoricalHealthEvidence( + input: Pick, + now: number, +): HistoricalHealthEvidence { try { openRequestHistoryIndexSync(); const handle = requestHistoryDb(); @@ -200,13 +249,25 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea ORDER BY timestamp DESC LIMIT ?`, ).all(...values, HEALTH_MAX_SAMPLES) as HealthRow[]; // Rows whose top-level target differs from this candidate may still carry - // candidate attempts (combo/failover): expand those too. + // candidate attempts (combo/failover): expand those too. The serialized + // provider/model LIKE prefilter keeps the LIMIT from being consumed by + // rows that cannot contribute samples for this candidate. + const escapeLike = (value: string): string => value.replace(/[\\%_]/g, match => `\\${match}`); const attemptRows = handle.query( `SELECT timestamp, attempt_count AS attemptCount, row_json AS rowJson FROM requests WHERE timestamp >= ? AND attempt_count > 1 + AND row_json LIKE ? ESCAPE '\\' + AND row_json LIKE ? ESCAPE '\\' AND NOT (provider = ? AND model = ?) ORDER BY timestamp DESC LIMIT ?`, - ).all(now - HEALTH_WINDOW_MS, input.provider, input.model, HEALTH_MAX_SAMPLES) as Array< + ).all( + now - HEALTH_WINDOW_MS, + `%\"provider\":\"${escapeLike(input.provider)}\"%`, + `%\"model\":\"${escapeLike(input.model)}\"%`, + input.provider, + input.model, + HEALTH_MAX_SAMPLES, + ) as Array< Pick >; @@ -253,18 +314,59 @@ export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHea } const sampleCount = successes + failures; + const out: HistoricalHealthEvidence = {}; if (sampleCount > 0) { - evidence.sampleCount = sampleCount; - evidence.successRate = weightedTotal > 0 ? weightedSuccess / weightedTotal : 0; - if (consecutiveFailures > 0) evidence.failures = consecutiveFailures; - if (incompleteStreams > 0) evidence.incompleteStreamRate = incompleteStreams / sampleCount; + out.sampleCount = sampleCount; + out.successRate = weightedTotal > 0 ? weightedSuccess / weightedTotal : 0; + if (consecutiveFailures > 0) out.failures = consecutiveFailures; + if (incompleteStreams > 0) out.incompleteStreamRate = incompleteStreams / sampleCount; latencies.sort((a, b) => a - b); const p50 = median(latencies); - if (p50 !== undefined) evidence.recentLatencyMs = p50; - evidence.recencyWeight = decayWeight(samples[0]!.timestamp, now); + if (p50 !== undefined) out.recentLatencyMs = p50; + out.recencyWeight = decayWeight(samples[0]!.timestamp, now); } + return out; } catch { /* index unreadable: evidence stays unknown */ + return {}; + } +} + +export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHealthEvidence { + const now = input.now ?? Date.now(); + const evidence: RouteHealthEvidence = {}; + + // Live authoritative state: hard cooldown and soft-avoid for Codex pool + // accounts. Cooldown stays authoritative over any historical score. Always + // read fresh - never cached. + if (input.codexAccountId && input.provider === "openai") { + if (isCodexAccountInCooldown(input.codexAccountId, now)) { + const until = getCodexAccountCooldownUntil(input.codexAccountId, now); + if (until !== null) evidence.cooldownUntilMs = until; + } + const softAvoidUntil = getCodexAccountSoftAvoidUntil(input.codexAccountId, now); + if (softAvoidUntil !== null && softAvoidUntil > now) evidence.softAvoidUntilMs = softAvoidUntil; + } + + const cacheKey = healthHistoryCacheKey(input); + const cached = healthHistoryCache.get(cacheKey); + if (cached && now - cached.at < HEALTH_HISTORY_CACHE_TTL_MS) { + Object.assign(evidence, cached.value); + } else { + const history = computeHistoricalHealthEvidence(input, now); + Object.assign(evidence, history); + if (healthHistoryCache.size >= HEALTH_HISTORY_CACHE_MAX_ENTRIES) { + let oldestKey: string | null = null; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [key, entry] of healthHistoryCache) { + if (entry.at < oldestAt) { + oldestAt = entry.at; + oldestKey = key; + } + } + if (oldestKey !== null) healthHistoryCache.delete(oldestKey); + } + healthHistoryCache.set(cacheKey, { at: now, value: history }); } return evidence; diff --git a/src/routing/profile.ts b/src/routing/profile.ts index d3b1f15a61..bc5ac85c39 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -153,15 +153,6 @@ function aliasIssues( if (hasOwnProvider(config.providers, alias)) { issues.push({ path: ["alias"], message: `alias "${alias}" collides with configured provider name "${alias}"` }); } - // A slashed alias whose first segment is a configured provider shadows that - // provider's routes (e.g. `a/m1` resolves as the alias before the provider - // namespace). Reject those aliases to preserve existing routing. - if (alias.includes("/") && hasOwnProvider(config.providers, alias.split("/")[0])) { - issues.push({ - path: ["alias"], - message: `alias "${alias}" collides with configured provider namespace "${alias.split("/")[0]}"`, - }); - } if (resolveComboId({ combos: config.combos }, alias)) { issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured combo selector` }); } diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 512390d3ee..658c66b1ad 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -208,6 +208,14 @@ export interface TraceBuildInput { function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; exclusions?: true }): RouteCandidateTrace { const exclusions = input.exclusions.slice(0, MAX_EXCLUSIONS_PER_CANDIDATE); if (exclusions.length < input.exclusions.length) budget.exclusions = true; + // Evidence reaches the builder from internal producers (bounded) or from + // caller-supplied dry-run input (unbounded). Whitelist + bound it through + // the same parsers the persisted-row normalizer uses so no unknown nested + // field or oversized string survives into the trace. + const capability = input.capability ? parseCapability(input.capability, budget) : undefined; + const health = input.health ? parseHealth(input.health) : undefined; + const quota = input.quota ? parseQuota(input.quota, budget) : undefined; + const cost = input.cost ? parseCost(input.cost, budget) : undefined; return { provider: capString(input.provider, budget), model: capString(input.model, budget), @@ -222,10 +230,10 @@ function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; ex : {}), })), ...(input.score ? { score: input.score } : {}), - ...(input.capability ? { capability: input.capability } : {}), - ...(input.health ? { health: input.health } : {}), - ...(input.quota ? { quota: input.quota } : {}), - ...(input.cost ? { cost: input.cost } : {}), + ...(capability ? { capability } : {}), + ...(health ? { health } : {}), + ...(quota ? { quota } : {}), + ...(cost ? { cost } : {}), }; } diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index 4e3301a1ae..b3fa6ec0bf 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -9,7 +9,7 @@ import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from "../../routing/profile"; import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; import { candidateCapabilityEvidence } from "../../routing/capability"; -import { healthEvidenceForCandidate } from "../../routing/health"; +import { policyCandidateHealthEvidence } from "../../routing/health"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; @@ -95,7 +95,8 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis if (!profile) { return jsonResponse({ error: { code: "missing_profile", message: "profile is required" } }, 400, req, config); } - if (!getRoutingProfile(config, profile)) { + const resolvedProfile = getRoutingProfile(config, profile); + if (!resolvedProfile) { return jsonResponse({ error: { code: "unknown_profile", message: `unknown routing profile: ${profile}` } }, 404, req, config); } const { evidence, ok } = parseEvidence(body.evidence); @@ -106,11 +107,11 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis // Match execution: fill the same candidate evidence the router would // assemble, so dry-run reports the same eligibility as real routing // instead of treating every capability as unknown. - ? getRoutingProfile(config, profile)!.candidates.map(candidate => ({ + ? resolvedProfile.candidates.map(candidate => ({ provider: candidate.provider, model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), - health: healthEvidenceForCandidate({ provider: candidate.provider, model: candidate.model }), + health: policyCandidateHealthEvidence(config, candidate), })) : parseCandidateEvidence(body.candidates); if (candidateEvidence === null) { diff --git a/tests/health-scoring.test.ts b/tests/health-scoring.test.ts index b14f8175ec..7e02496490 100644 --- a/tests/health-scoring.test.ts +++ b/tests/health-scoring.test.ts @@ -4,7 +4,13 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { appendUsageEntry, resetUsageReadCacheForTests, type PersistedUsageEntry } from "../src/usage/log"; import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; -import { healthEvidenceForCandidate, healthScore, HEALTH_SCORE_CONSTANTS } from "../src/routing/health"; +import { + clearHealthHistoryCacheForTests, + codexPoolHealthEvidence, + healthEvidenceForCandidate, + healthScore, + HEALTH_SCORE_CONSTANTS, +} from "../src/routing/health"; import { evaluatePolicyProfile } from "../src/routing/evaluator"; import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; import type { OcxConfig } from "../src/types"; @@ -35,6 +41,7 @@ beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-health-")); process.env.OPENCODEX_HOME = testDir; resetUsageReadCacheForTests(); + clearHealthHistoryCacheForTests(); closeRequestHistoryIndex(); }); @@ -266,4 +273,54 @@ describe("health-aware scoring (RI-06)", () => { expect(route.providerName).toBe("b"); expect(route.modelId).toBe("m2"); }); + + test("mixed pool cooldown/soft-avoid states degrade to soft-avoid", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const cfg = config({ + codexAccounts: [ + { id: "pool-a", email: "pool-a@example.test", isMain: false }, + { id: "pool-b", email: "pool-b@example.test", isMain: false }, + ], + }); + // pool-a hard-cooled (429); pool-b soft-avoided (transient 503s). No + // single account is usable, so the aggregate must degrade to soft-avoid. + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { retryAfter: "3600", now }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now: now + 1 }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now: now + 2 }); + const evidence = codexPoolHealthEvidence(cfg, now + 3); + expect(evidence?.softAvoidUntilMs).toBeDefined(); + expect(evidence?.cooldownUntilMs).toBeUndefined(); + }); + + test("unknown health under allow blends neutrally instead of outranking measured health", () => { + const cfg = config({ + routingProfiles: { + ranking: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + optimize: { latency: 0, health: 0.8, cost: 0, quota: 0 }, + unknownEvidence: { capability: "allow", health: "allow", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const result = evaluatePolicyProfile(cfg, "ranking", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + { + provider: "b", + model: "m2", + capability: { contextWindow: 200000 }, + health: { sampleCount: 50, successRate: 1, recentLatencyMs: 100 }, + }, + ]); + // a (priority 1.0, unknown -> neutral 0.5) blends to 0.5; b (priority + // 0.5, near-perfect health) blends above it. Without the neutral blend the + // unknown candidate would outrank the measured one. + expect(result.selectedIndex).toBe(1); + expect(result.candidates[0]!.score!.components.health).toBe(0.5); + }); }); diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index a4dbef1585..77570fb96f 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -174,6 +174,22 @@ describe("policy execution (RI-05)", () => { expect(route.routeDecision!.candidates[0]!.capability?.tools).toBe(true); }); + test("kiro and mimo-free adapters infer tool support", () => { + const config = baseConfig({ + providers: { + ...baseConfig().providers, + k: { adapter: "kiro", baseUrl: "https://k.example/v1", apiKey: "kk", models: ["m9"] }, + m: { adapter: "mimo-free", baseUrl: "https://m.example/v1", apiKey: "km", models: ["m8"] }, + }, + routingProfiles: { + ktools: { candidates: [{ provider: "k", model: "m9" }], require: { tools: true } }, + mtools: { candidates: [{ provider: "m", model: "m8" }], require: { tools: true } }, + }, + }); + expect(routeModel(config, "policy/ktools")).toMatchObject({ providerName: "k", modelId: "m9" }); + expect(routeModel(config, "policy/mtools")).toMatchObject({ providerName: "m", modelId: "m8" }); + }); + test("request evidence constrains candidates: image input excludes non-image models", () => { const config = baseConfig({ routingProfiles: { diff --git a/tests/route-decision-trace.test.ts b/tests/route-decision-trace.test.ts index cb6fd6a599..5a0c005796 100644 --- a/tests/route-decision-trace.test.ts +++ b/tests/route-decision-trace.test.ts @@ -274,6 +274,34 @@ describe("route decision traces (RI-01)", () => { expect(trace.truncated?.strings).toBe(true); }); + test("caller-supplied evidence is whitelisted and bounded in the trace", () => { + const trace = buildRouteDecisionTrace({ + requestedModel: "policy/p", + routeKind: "policy", + candidates: [{ + provider: "a", + model: "m1", + eligible: true, + exclusions: [], + capability: { + serviceTier: "x".repeat(500), + reasoningEfforts: ["y".repeat(500)], + unknownNested: { z: "should-not-survive" }, + }, + quota: { known: true, source: "s".repeat(300) }, + cost: { estimatedUsd: 0.5, priceSource: "p".repeat(300) }, + }], + selected: { provider: "a", model: "m1", reason: "policy-selected" }, + }); + const candidate = trace.candidates[0]!; + expect(candidate.capability?.serviceTier?.length).toBe(MAX_TRACE_STRING); + expect(candidate.capability?.reasoningEfforts?.[0]?.length).toBe(MAX_TRACE_STRING); + expect(candidate.capability && "unknownNested" in candidate.capability).toBe(false); + expect(candidate.quota?.source?.length).toBe(MAX_TRACE_STRING); + expect(candidate.cost?.priceSource?.length).toBe(MAX_TRACE_STRING); + expect(trace.truncated?.strings).toBe(true); + }); + test("trace never contains credentials or prompt content", () => { const config = baseConfig(); // Built at runtime so the privacy scanner's key-pattern grep does not diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index acb9046a96..c7ddc9e02d 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -156,7 +156,13 @@ describe("routing profiles (RI-04)", () => { candidates: [{ provider: "a", model: "m1" }], alias: "a/m1", }, config); - expect(providerNamespaceCollision.some(issue => issue.message.includes("provider routing namespace"))).toBe(true); + // Exactly one issue: the first-segment provider collision must not be + // reported twice with different wordings. + const namespaceIssues = providerNamespaceCollision.filter( + issue => issue.message.includes("provider routing namespace"), + ); + expect(namespaceIssues.length).toBe(1); + expect(providerNamespaceCollision.length).toBe(1); const siblingCollision = routingProfileIssues("p", { candidates: [{ provider: "a", model: "m1" }], @@ -401,4 +407,33 @@ describe("routing profiles (RI-04)", () => { expect(body.candidates?.[0]).toMatchObject({ provider: "a", eligible: true }); expect(body.candidates?.[1]).toMatchObject({ provider: "b", eligible: false }); }); + + test("API dry-run mirrors live codex cooldown for openai candidates", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const config = baseConfig({ + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"], modelContextWindows: { m1: 200_000 }, parallelToolCalls: true }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + only: { candidates: [{ provider: "openai", model: "gpt-5.6" }] }, + }, + }); + recordCodexUpstreamOutcome(config, "pool-a", 429, { retryAfter: "3600", now }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "only", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { candidates?: Array<{ eligible?: boolean; exclusions?: Array<{ code: string }> }> }; + expect(body.candidates?.[0]?.eligible).toBe(false); + expect(body.candidates?.[0]?.exclusions?.some(exclusion => exclusion.code === "cooldown")).toBe(true); + }); }); From 6df2ec74a169a20464fb16592bd2ce97427ae6bf Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:02:44 +0200 Subject: [PATCH 10/10] docs(devlog): record RI-06 synced head sha --- devlog/_plan/260804_router_intelligence/001_pr_stack_status.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 6c7117e704..c0e6ebf872 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -45,7 +45,7 @@ other; closing one is a maintainer decision and neither is stale. | RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | `a594938c5` | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | MERGED | | RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | MERGED | | RI-05 | `feat/ri-05-capability-aware-routing` | `dev` (post-#1011 merge) | `088194a3a` | #1012 | https://github.com/lidge-jun/opencodex/pull/1012 | MERGED | -| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | pending (post-sync head) | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | in progress | +| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | in progress | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | | RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | | RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued |