diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index ed6eff696..c0e6ebf87 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -43,9 +43,9 @@ other; closing one is a maintainer decision and neither is stale. | RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | MERGED | | RI-02 | `feat/ri-02-request-history-index` | `dev` (post-#1003 merge) | `2a72aa4a9` | #1004 | https://github.com/lidge-jun/opencodex/pull/1004 | MERGED | | RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | `a594938c5` | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | MERGED | -| RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | OPEN (resync) | -| RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | -| RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | +| RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | MERGED | +| RI-05 | `feat/ri-05-capability-aware-routing` | `dev` (post-#1011 merge) | `088194a3a` | #1012 | https://github.com/lidge-jun/opencodex/pull/1012 | MERGED | +| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | in progress | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | | RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | | RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued | @@ -179,3 +179,38 @@ other; closing one is a maintainer decision and neither is stale. identically on the pristine base (Windows symlink EPERM, environmental) - Remaining Low findings: none (B3/B5 residual: no-eligible trace names candidate 0 as `selected`; API evidence fields are permissively dropped) + +### RI-05 - feat/ri-05-capability-aware-routing + +- PR: #1012 (MERGED) https://github.com/lidge-jun/opencodex/pull/1012 +- Merged on `dev` at `088194a3a` (2026-08-05). Includes the RI-05 execution + wiring, request-evidence hardening, no-eligible trace persistence, and + policy-namespace reservation. + +### RI-06 - feat/ri-06-health-aware-routing + +- Base SHA: `56f17f45c` (RI-05 head); PR #1013 https://github.com/lidge-jun/opencodex/pull/1013. +- Findings (self-review): 4 fixed pre-push - (1) route-time health evidence + needed synchronous index access (`openRequestHistoryIndexSync`); (2) + unknown-health "penalize" folds a deterministic 0.3 floor; (3) trace + candidates carry capability/health/quota/cost evidence; (4) score + assertions updated for the health component. +- Full-review round (verdict `changes-requested`, all items fixed): + 1. indexer: dev/ino identity (already on dev) + row validation + clean-tail + `lastError`; appends are a tail, never a rebuild; + 2. router: live Codex pool cooldown/soft-avoid evidence for `openai` + targets (`codexPoolHealthEvidence` + active account); + 3. health: combo/failover `attempts[]` expand into per-target samples; + 4. request evidence: nested message `content` arrays walked for images + (already on dev via #1012); + 5. evaluator: request-side `contextWindow`/`structuredOutputRequired`/ + `encryptedCodexTask` enforced (already on dev via #1012); + 6. dry-run API: omitted `candidates` populate execution-equivalent + evidence; + 7. alias validation rejects first-segment provider aliases; + 8. `policy/` without a profile falls through to normal resolution; + 9. capability: adapter-level tool inference for tool-capable adapters. +- 8 bot threads fixed + resolved; `tsc --noEmit` 0 errors; routing suites + green; `privacy:scan` passed. +- Sync: merged `dev` (post-#1012, `088194a3a`) into the branch so the head is + mergeable and CI can run; base sync of the stack continues with RI-07. diff --git a/src/router.ts b/src/router.ts index 9ced30e83..61a25b845 100644 --- a/src/router.ts +++ b/src/router.ts @@ -31,6 +31,7 @@ import { import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; import { candidateCapabilityEvidence } from "./routing/capability"; +import { policyCandidateHealthEvidence } from "./routing/health"; export class NoEligiblePolicyCandidateError extends Error { /** Evaluation trace (with per-candidate exclusions) when nothing qualified. */ @@ -489,17 +490,24 @@ function routeModelInternal( const slash = modelId.indexOf("/"); // Policy namespace is system-reserved: an explicit `policy/` or a // configured profile alias executes the policy evaluator and routes the - // selected candidate. Only explicit requests reach this branch. - const policyId = resolvePolicyProfileId(config, modelId); - if (policyId) { - const profile = getRoutingProfile(config, policyId); - if (!profile) throw new Error(`Unknown routing profile: ${policyId}`); + // selected candidate. Only explicit requests reach this branch; concrete + // recursive targets skip policy resolution entirely (bypassCombos) so an + // alias matching a selected candidate can never recurse, and a + // `policy/` without a configured profile falls through to normal + // provider/default resolution instead of failing. + const policyId = !bypassCombos ? resolvePolicyProfileId(config, modelId) : null; + const profile = policyId ? getRoutingProfile(config, policyId) : undefined; + if (profile && policyId) { + // One clock read per decision keeps candidate evidence, exclusions, and + // scores mutually consistent and reproducible. + const now = Date.now(); const candidateEvidence = profile.candidates.map(candidate => ({ provider: candidate.provider, model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + health: policyCandidateHealthEvidence(config, candidate, now), })); - const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); + const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence, now); if (evaluation.selectedIndex === null) { throw new NoEligiblePolicyCandidateError(policyId, evaluation.trace); } diff --git a/src/routing/capability.ts b/src/routing/capability.ts index ae9919f37..b628e55df 100644 --- a/src/routing/capability.ts +++ b/src/routing/capability.ts @@ -96,6 +96,23 @@ function classifyHostname(hostname: string): "local" | "private" | null { return null; } +/** + * Adapters whose upstream protocol supports function/tool calling. Mirrors + * the adapter ids the resolver accepts (including the `azure` alias for + * `azure-openai`); `kiro` and `mimo-free` send/delegate tool calls. + */ +const TOOL_CAPABLE_ADAPTERS = new Set([ + "openai-chat", + "openai-responses", + "anthropic", + "cursor", + "google", + "azure-openai", + "azure", + "kiro", + "mimo-free", +]); + function localRemoteEvidence(baseUrl: string | undefined): Pick { if (typeof baseUrl !== "string" || baseUrl.length === 0) return {}; try { @@ -144,11 +161,16 @@ export function candidateCapabilityEvidence( : undefined; const capabilities = catalogRow?.capabilities ?? []; - // `parallelToolCalls` is provider-level evidence that the provider accepts - // parallel tool calls (registry-set per provider); the catalog `capabilities` - // list is the per-model signal. Both are positive local evidence only. + // The catalog `capabilities` list is a positive per-model signal; a row + // without "tools" is treated as unknown, never as a negative. Without a + // catalog row the adapter protocol itself is the signal: tool-capable + // adapters run single tool calls even when the parallel-call opt-in is + // unset or false. `parallelToolCalls` stays a positive provider-level + // override. const tools = capabilities.includes("tools") - || (isNative ? true : provider?.parallelToolCalls === true) + || isNative + || (catalogRow === undefined && provider !== undefined && TOOL_CAPABLE_ADAPTERS.has(provider.adapter)) + || provider?.parallelToolCalls === true || undefined; const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId] diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 6ecf4fc61..363f97d06 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -21,6 +21,12 @@ import { type Unknownable, } from "./trace"; import { getRoutingProfile, policyModelId, type NormalizedRoutingProfile } from "./profile"; +import { healthScore } from "./health"; + +/** Unknown health under "penalize": a low-but-not-zero deterministic floor. */ +export const HEALTH_UNKNOWN_PENALTY_SCORE = 0.3; +/** Unknown health under "allow": neutral midpoint of the [0,1] health scale. */ +export const HEALTH_UNKNOWN_NEUTRAL_SCORE = 0.5; export interface PolicyRequestEvidence { /** Required context window for this request (tokens). */ @@ -238,6 +244,7 @@ export function evaluatePolicyProfile( profileId: string, requestEvidence: PolicyRequestEvidence, candidateEvidence: PolicyCandidateEvidence[], + now = Date.now(), ): PolicyEvaluationResult { const profile = getRoutingProfile(config, profileId); if (!profile) throw new Error(`Unknown routing profile: ${profileId}`); @@ -279,13 +286,36 @@ export function evaluatePolicyProfile( if (overCostLimit) { exclusions.push({ code: "cost-limit", detail: "maxEstimatedCostUsd" }); } - const eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + + // Health scoring (RI-06): live hard cooldown is authoritative and + // excludes; unknown health follows the profile's unknownEvidence policy; + // historical health never overrides explicit ineligibility. + const health = evidence.health; + let healthValue = health ? healthScore(health, now) : null; + if (health?.cooldownUntilMs !== undefined && health.cooldownUntilMs > now) { + exclusions.push({ code: "cooldown" }); + eligible = false; + } else if (healthValue === null && profile.unknownEvidence.health === "exclude") { + exclusions.push({ code: "unknown-health" }); + eligible = false; + } else if (healthValue === null && profile.unknownEvidence.health === "penalize") { + healthValue = HEALTH_UNKNOWN_PENALTY_SCORE; + } else if (healthValue === null && profile.unknownEvidence.health === "allow") { + // Neutral midpoint: blending keeps an unknown candidate from outranking + // a measured one with the same configured priority. + healthValue = HEALTH_UNKNOWN_NEUTRAL_SCORE; + } const priorityScore = configuredPriorityScore(index, profile.candidates.length); - const score: RouteScoreEvidence = { - total: priorityScore, - components: { configuredPriority: priorityScore }, - }; + const healthWeight = profile.optimize.health; + const components: RouteScoreEvidence["components"] = { configuredPriority: priorityScore }; + let total = priorityScore; + if (healthWeight > 0 && healthValue !== null) { + total = priorityScore * (1 - healthWeight) + healthValue * healthWeight; + components.health = healthValue; + } + const score: RouteScoreEvidence = { total, components }; const evaluated: PolicyEvaluationCandidate = { provider: evidence.provider, model: evidence.model, @@ -321,6 +351,10 @@ export function evaluatePolicyProfile( eligible: candidate.eligible, exclusions: candidate.exclusions, ...(candidate.score ? { score: candidate.score } : {}), + ...(candidate.capability ? { capability: candidate.capability } : {}), + ...(candidate.health ? { health: candidate.health } : {}), + ...(candidate.quota ? { quota: candidate.quota } : {}), + ...(candidate.cost ? { cost: candidate.cost } : {}), })), selected: selectedIndex === null ? { provider: candidates[0]?.provider ?? "", model: candidates[0]?.model ?? "", reason: "no-eligible-candidate" } diff --git a/src/routing/health.ts b/src/routing/health.ts new file mode 100644 index 000000000..f1bee547e --- /dev/null +++ b/src/routing/health.ts @@ -0,0 +1,401 @@ +/** + * Evidence-based route health (RI-06). + * + * Health evidence combines: + * - live in-memory routing state: Codex account cooldown / soft-avoid + * (authoritative hard state); + * - historical evidence from the request-history index: success rate, + * consecutive failures, incomplete-stream rate, recent latency, sample + * count, recency-decayed weights. + * + * Failure classification is strict: client cancellations, invalid requests + * (4xx except quota 429) and synthetic policy refusals never damage target + * health. Transport-neutral failures are excluded by the classification the + * routing layer already records (host/account split per #914 work). + * + * All formulas are deterministic with documented constants; no ML. + */ + +import type { OcxConfig } from "../types"; +import { openRequestHistoryIndexSync, requestHistoryDb } from "./history/indexer"; +import { OPENAI_CODEX_PROVIDER_ID } from "../providers/openai-tiers"; +import { + getCodexAccountCooldownUntil, + getCodexAccountSoftAvoidUntil, + getEffectiveActiveCodexAccountId, + isCodexAccountInCooldown, + listLiveCodexAccountIds, +} from "../codex/routing"; +import type { RouteHealthEvidence } from "./trace"; + +export const HEALTH_SCORE_CONSTANTS = { + /** Recent-success weight in the composite. */ + SUCCESS_WEIGHT: 0.50, + /** Incomplete-stream (negative) weight. */ + INCOMPLETE_WEIGHT: 0.15, + /** Recent-latency weight. */ + LATENCY_WEIGHT: 0.20, + /** Consecutive-failure recovery weight. */ + RECOVERY_WEIGHT: 0.15, + /** p50 latency at or above this (ms) scores zero on the latency axis. */ + LATENCY_TARGET_MS: 60_000, + /** Samples needed for full confidence; fewer samples scale the score down. */ + MIN_CONFIDENCE_SAMPLES: 20, + /** Soft-avoid multiplies the composite. */ + SOFT_AVOID_MULTIPLIER: 0.5, + /** Recency decay: a sample loses half its weight every RECENCY_HALF_LIFE_DAYS. */ + RECENCY_HALF_LIFE_DAYS: 7, +} as const; + +export const HEALTH_WINDOW_MS = 14 * 86_400_000; +export const HEALTH_MAX_SAMPLES = 100; + +/** + * Historical health evidence is cached briefly across candidates within one + * routing decision (and rapid successive decisions). Live cooldown/soft-avoid + * state is always read fresh - it is never cached. The TTL bounds how stale + * the history index read may be; 1.5s keeps routing deterministic within a + * decision while bounding per-candidate SQLite work. + */ +const HEALTH_HISTORY_CACHE_TTL_MS = 1_500; +const HEALTH_HISTORY_CACHE_MAX_ENTRIES = 64; + +type HistoricalHealthEvidence = Pick< + RouteHealthEvidence, + "sampleCount" | "successRate" | "failures" | "incompleteStreamRate" | "recentLatencyMs" | "recencyWeight" +>; + +const healthHistoryCache = new Map(); + +/** Test seam: routing tests append fresh rows and must not see cached history. */ +export function clearHealthHistoryCacheForTests(): void { + healthHistoryCache.clear(); +} + +function healthHistoryCacheKey(input: Pick): string { + return `${input.provider}\u0000${input.model}\u0000${input.accountRef ?? ""}`; +} + +export interface HealthEvidenceInput { + provider: string; + model: string; + accountRef?: string; + /** Live codex account id for cooldown/soft-avoid state (provider "openai"). */ + codexAccountId?: string; + now?: number; +} + +interface HealthSample { + status: number; + closeReason: string | null; + terminalStatus: string | null; + durationMs: number; + timestamp: number; +} + +interface HealthRow extends HealthSample { + attemptCount?: number; + rowJson?: string | null; +} + +/** + * Per-attempt samples for a candidate from a row's persisted entry. + * Combo/failover requests store each upstream try in `entry.attempts` while + * the top-level row records the final outcome; a provider/model that failed + * as a non-final attempt must still contribute its own health samples. + */ +function attemptSamplesFor( + row: Pick, + provider: string, + model: string, +): HealthSample[] { + if (!row.rowJson || (row.attemptCount ?? 1) <= 1) return []; + try { + const parsed = JSON.parse(row.rowJson) as { attempts?: unknown }; + if (!Array.isArray(parsed.attempts)) return []; + const samples: HealthSample[] = []; + for (const attempt of parsed.attempts) { + if (!attempt || typeof attempt !== "object" || Array.isArray(attempt)) continue; + const record = attempt as Record; + if (record.provider !== provider || record.model !== model) continue; + if (typeof record.status !== "number" || typeof record.durationMs !== "number") continue; + samples.push({ + status: record.status, + closeReason: null, + terminalStatus: null, + durationMs: record.durationMs, + timestamp: row.timestamp, + }); + } + return samples; + } catch { + return []; + } +} + +/** + * Live Codex pool account state for an `openai` policy candidate. + * + * When a deterministic active account exists (manual selection or + * `config.activeCodexAccountId`) its cooldown/soft-avoid state is + * authoritative. Otherwise the target is conservatively cooled only when + * every live pool account is cooling or soft-avoided; account selection + * inside `src/codex/routing.ts` stays authoritative when any account is + * usable, so policy scoring never invents account choices. + */ +export function codexPoolHealthEvidence( + config: Parameters[0], + now = Date.now(), +): Pick | undefined { + const activeId = getEffectiveActiveCodexAccountId(config); + if (activeId) { + const until = getCodexAccountCooldownUntil(activeId, now); + if (until !== null) return { cooldownUntilMs: until }; + const softUntil = getCodexAccountSoftAvoidUntil(activeId, now); + if (softUntil !== null && softUntil > now) return { softAvoidUntilMs: softUntil }; + return undefined; + } + const live = [...listLiveCodexAccountIds(config)]; + if (live.length === 0) return undefined; + const cooldowns: number[] = []; + const softAvoids: number[] = []; + for (const accountId of live) { + const until = getCodexAccountCooldownUntil(accountId, now); + if (until !== null) cooldowns.push(until); + const softUntil = getCodexAccountSoftAvoidUntil(accountId, now); + if (softUntil !== null && softUntil > now) softAvoids.push(softUntil); + } + if (cooldowns.length === live.length) return { cooldownUntilMs: Math.max(...cooldowns) }; + // Every account is unavailable, but not uniformly hard-cooled (e.g. some in + // cooldown, some soft-avoided): degrade to soft-avoid with the latest expiry + // so scoring still penalizes the pool. + if (cooldowns.length + softAvoids.length >= live.length && softAvoids.length > 0) { + return { softAvoidUntilMs: Math.max(...softAvoids, ...cooldowns) }; + } + return undefined; +} + +/** + * Candidate health evidence assembled exactly like the router's policy path: + * historical evidence plus authoritative live Codex pool state for `openai` + * targets. Shared by the router and the dry-run management route so the two + * surfaces cannot drift apart. + */ +export function policyCandidateHealthEvidence( + config: Parameters[0], + candidate: { provider: string; model: string }, + now = Date.now(), +): RouteHealthEvidence { + return { + ...healthEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? getEffectiveActiveCodexAccountId(config) + : undefined, + now, + }), + // Live pool state stays authoritative for `openai` targets even when no + // account reference exists in the candidate evidence. + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? (codexPoolHealthEvidence(config, now) ?? {}) + : {}), + }; +} + +function classifySample(sample: HealthSample): "success" | "failure" | "neutral" { + if (sample.closeReason === "client_cancel" || sample.status === 499) return "neutral"; + // Invalid requests and policy refusals must not poison target health. + if (sample.status >= 400 && sample.status < 500 && sample.status !== 429) return "neutral"; + if (sample.terminalStatus === "incomplete") return "failure"; + if (sample.terminalStatus && sample.terminalStatus !== "completed") return "failure"; + if (sample.status >= 400) return "failure"; + return "success"; +} + +function decayWeight(timestamp: number, now: number): number { + const ageDays = Math.max(0, now - timestamp) / 86_400_000; + return Math.pow(0.5, ageDays / HEALTH_SCORE_CONSTANTS.RECENCY_HALF_LIFE_DAYS); +} + +function median(sorted: number[]): number | undefined { + if (sorted.length === 0) return undefined; + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; +} + +/** + * Historical health evidence from the derived index (synchronous: called at + * routing time). Never throws; an unopened/unreadable index yields unknown. + */ +function computeHistoricalHealthEvidence( + input: Pick, + now: number, +): HistoricalHealthEvidence { + try { + openRequestHistoryIndexSync(); + const handle = requestHistoryDb(); + const where: string[] = ["provider = ?", "model = ?", "timestamp >= ?"]; + const values: Array = [input.provider, input.model, now - HEALTH_WINDOW_MS]; + if (input.accountRef) { + where.push("api_key_id = ?"); + values.push(input.accountRef); + } + const rows = handle.query( + `SELECT status, close_reason AS closeReason, terminal_status AS terminalStatus, + duration_ms AS durationMs, timestamp, + attempt_count AS attemptCount, row_json AS rowJson + FROM requests WHERE ${where.join(" AND ")} + ORDER BY timestamp DESC LIMIT ?`, + ).all(...values, HEALTH_MAX_SAMPLES) as HealthRow[]; + // Rows whose top-level target differs from this candidate may still carry + // candidate attempts (combo/failover): expand those too. The serialized + // provider/model LIKE prefilter keeps the LIMIT from being consumed by + // rows that cannot contribute samples for this candidate. + const escapeLike = (value: string): string => value.replace(/[\\%_]/g, match => `\\${match}`); + const attemptRows = handle.query( + `SELECT timestamp, attempt_count AS attemptCount, row_json AS rowJson + FROM requests WHERE timestamp >= ? AND attempt_count > 1 + AND row_json LIKE ? ESCAPE '\\' + AND row_json LIKE ? ESCAPE '\\' + AND NOT (provider = ? AND model = ?) + ORDER BY timestamp DESC LIMIT ?`, + ).all( + now - HEALTH_WINDOW_MS, + `%\"provider\":\"${escapeLike(input.provider)}\"%`, + `%\"model\":\"${escapeLike(input.model)}\"%`, + input.provider, + input.model, + HEALTH_MAX_SAMPLES, + ) as Array< + Pick + >; + + const samples: HealthSample[] = []; + for (const row of rows) { + const attemptSamples = attemptSamplesFor(row, input.provider, input.model); + samples.push(...(attemptSamples.length > 0 ? attemptSamples : [row])); + } + for (const row of attemptRows) { + samples.push(...attemptSamplesFor(row, input.provider, input.model)); + } + // Newest first for the consecutive-failure walk; attempt samples inherit + // their row's timestamp. + samples.sort((a, b) => b.timestamp - a.timestamp); + + let successes = 0; + let failures = 0; + let incompleteStreams = 0; + let weightedSuccess = 0; + let weightedTotal = 0; + const latencies: number[] = []; + for (const sample of samples) { + const kind = classifySample(sample); + const weight = decayWeight(sample.timestamp, now); + if (kind === "neutral") continue; + if (kind === "success") { + successes += 1; + weightedSuccess += weight; + weightedTotal += weight; + } else { + failures += 1; + weightedTotal += weight; + } + if (sample.terminalStatus === "incomplete") incompleteStreams += 1; + latencies.push(sample.durationMs); + } + // Consecutive failures: walk newest -> oldest until a success. + let consecutiveFailures = 0; + for (const sample of samples) { + const kind = classifySample(sample); + if (kind === "neutral") continue; + if (kind === "failure") consecutiveFailures += 1; + else break; + } + + const sampleCount = successes + failures; + const out: HistoricalHealthEvidence = {}; + if (sampleCount > 0) { + out.sampleCount = sampleCount; + out.successRate = weightedTotal > 0 ? weightedSuccess / weightedTotal : 0; + if (consecutiveFailures > 0) out.failures = consecutiveFailures; + if (incompleteStreams > 0) out.incompleteStreamRate = incompleteStreams / sampleCount; + latencies.sort((a, b) => a - b); + const p50 = median(latencies); + if (p50 !== undefined) out.recentLatencyMs = p50; + out.recencyWeight = decayWeight(samples[0]!.timestamp, now); + } + return out; + } catch { + /* index unreadable: evidence stays unknown */ + return {}; + } +} + +export function healthEvidenceForCandidate(input: HealthEvidenceInput): RouteHealthEvidence { + const now = input.now ?? Date.now(); + const evidence: RouteHealthEvidence = {}; + + // Live authoritative state: hard cooldown and soft-avoid for Codex pool + // accounts. Cooldown stays authoritative over any historical score. Always + // read fresh - never cached. + if (input.codexAccountId && input.provider === "openai") { + if (isCodexAccountInCooldown(input.codexAccountId, now)) { + const until = getCodexAccountCooldownUntil(input.codexAccountId, now); + if (until !== null) evidence.cooldownUntilMs = until; + } + const softAvoidUntil = getCodexAccountSoftAvoidUntil(input.codexAccountId, now); + if (softAvoidUntil !== null && softAvoidUntil > now) evidence.softAvoidUntilMs = softAvoidUntil; + } + + const cacheKey = healthHistoryCacheKey(input); + const cached = healthHistoryCache.get(cacheKey); + if (cached && now - cached.at < HEALTH_HISTORY_CACHE_TTL_MS) { + Object.assign(evidence, cached.value); + } else { + const history = computeHistoricalHealthEvidence(input, now); + Object.assign(evidence, history); + if (healthHistoryCache.size >= HEALTH_HISTORY_CACHE_MAX_ENTRIES) { + let oldestKey: string | null = null; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [key, entry] of healthHistoryCache) { + if (entry.at < oldestAt) { + oldestAt = entry.at; + oldestKey = key; + } + } + if (oldestKey !== null) healthHistoryCache.delete(oldestKey); + } + healthHistoryCache.set(cacheKey, { at: now, value: history }); + } + + return evidence; +} + +/** + * Deterministic health score in [0,1]. Returns null when evidence is unknown + * (no samples) so callers can apply the profile's unknownEvidence policy. + * A live hard cooldown scores 0 (authoritative). + */ +export function healthScore(evidence: RouteHealthEvidence | undefined, now = Date.now()): number | null { + if (!evidence) return null; + if (evidence.cooldownUntilMs !== undefined && evidence.cooldownUntilMs > now) return 0; + if (!evidence.sampleCount || evidence.sampleCount < 1) return null; + const successRate = evidence.successRate ?? 0; + const incompleteRate = evidence.incompleteStreamRate ?? 0; + const p50 = evidence.recentLatencyMs; + const latencyScore = p50 === undefined + ? 0.5 + : Math.max(0, Math.min(1, 1 - p50 / HEALTH_SCORE_CONSTANTS.LATENCY_TARGET_MS)); + const consecutive = evidence.failures ?? 0; + const recoveryScore = 1 - Math.min(1, consecutive / 5); + const composite = HEALTH_SCORE_CONSTANTS.SUCCESS_WEIGHT * successRate + + HEALTH_SCORE_CONSTANTS.INCOMPLETE_WEIGHT * (1 - incompleteRate) + + HEALTH_SCORE_CONSTANTS.LATENCY_WEIGHT * latencyScore + + HEALTH_SCORE_CONSTANTS.RECOVERY_WEIGHT * recoveryScore; + const confidence = Math.min(1, evidence.sampleCount / HEALTH_SCORE_CONSTANTS.MIN_CONFIDENCE_SAMPLES); + const softAvoid = evidence.softAvoidUntilMs !== undefined && evidence.softAvoidUntilMs > now + ? HEALTH_SCORE_CONSTANTS.SOFT_AVOID_MULTIPLIER + : 1; + return composite * confidence * softAvoid; +} diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index b8c915de7..cad9ea5c5 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -215,7 +215,10 @@ function parsedEntryFromLine(line: string): PersistedUsageEntry | null { if (parsed && typeof parsed === "object" && typeof parsed.requestId === "string" && typeof parsed.timestamp === "number" - && typeof parsed.provider === "string") { + && typeof parsed.provider === "string" + && typeof parsed.model === "string" + && typeof parsed.status === "number" + && typeof parsed.durationMs === "number") { return parsed; } } catch { @@ -408,7 +411,7 @@ function fullRebuild(dbHandle: Database, reason: string): void { setMeta(dbHandle, HISTORY_META_KEYS.builtAtMs, Date.now()); } -async function refreshLocked(): Promise { +function refreshLockedSync(): RequestHistoryIndexMeta { openIndexDb(); const state = ensureSchemaAndIdentity(db!); const handle = db!; @@ -429,12 +432,20 @@ async function refreshLocked(): Promise { fullRebuild(handle, "source truncated; index rebuilt"); return metaFor(handle); } - if (tailNextOffset < Number(revision.size)) { - ingestSourceTail(handle, revision.path, tailNextOffset); - } + if (tailNextOffset < Number(revision.size)) { + const inserted = ingestSourceTail(handle, revision.path, tailNextOffset); + // A clean tail ingest proves the index is healthy: clear any earlier + // rebuild marker so status readers can distinguish rebuilds from tails. + if (inserted > 0) setMeta(handle, HISTORY_META_KEYS.lastError, ""); + } return metaFor(handle); } +/** Synchronous refresh for routing-time evidence reads (RI-06+). */ +export function openRequestHistoryIndexSync(): RequestHistoryIndexMeta { + return refreshLockedSync(); +} + /** * Open (and refresh) the index. Single-flight: concurrent callers share one * refresh. Never throws for missing/corrupt index or ledger state; those are @@ -442,7 +453,7 @@ async function refreshLocked(): Promise { */ export function openRequestHistoryIndex(): Promise { if (!openPromise) { - openPromise = refreshLocked().finally(() => { + openPromise = Promise.resolve(refreshLockedSync()).finally(() => { openPromise = null; }); } diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 7a97d421c..658c66b1a 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -180,6 +180,10 @@ export interface TraceCandidateInput { eligible: boolean; exclusions: RouteExclusionReason[]; score?: RouteScoreEvidence; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; } export interface TraceBuildInput { @@ -204,6 +208,14 @@ export interface TraceBuildInput { function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; exclusions?: true }): RouteCandidateTrace { const exclusions = input.exclusions.slice(0, MAX_EXCLUSIONS_PER_CANDIDATE); if (exclusions.length < input.exclusions.length) budget.exclusions = true; + // Evidence reaches the builder from internal producers (bounded) or from + // caller-supplied dry-run input (unbounded). Whitelist + bound it through + // the same parsers the persisted-row normalizer uses so no unknown nested + // field or oversized string survives into the trace. + const capability = input.capability ? parseCapability(input.capability, budget) : undefined; + const health = input.health ? parseHealth(input.health) : undefined; + const quota = input.quota ? parseQuota(input.quota, budget) : undefined; + const cost = input.cost ? parseCost(input.cost, budget) : undefined; return { provider: capString(input.provider, budget), model: capString(input.model, budget), @@ -218,6 +230,10 @@ function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; ex : {}), })), ...(input.score ? { score: input.score } : {}), + ...(capability ? { capability } : {}), + ...(health ? { health } : {}), + ...(quota ? { quota } : {}), + ...(cost ? { cost } : {}), }; } diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index 78cc0a18c..b3fa6ec0b 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -8,6 +8,8 @@ import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from "../../routing/profile"; import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; +import { candidateCapabilityEvidence } from "../../routing/capability"; +import { policyCandidateHealthEvidence } from "../../routing/health"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; @@ -93,7 +95,8 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis if (!profile) { return jsonResponse({ error: { code: "missing_profile", message: "profile is required" } }, 400, req, config); } - if (!getRoutingProfile(config, profile)) { + const resolvedProfile = getRoutingProfile(config, profile); + if (!resolvedProfile) { return jsonResponse({ error: { code: "unknown_profile", message: `unknown routing profile: ${profile}` } }, 404, req, config); } const { evidence, ok } = parseEvidence(body.evidence); @@ -101,7 +104,15 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config); } const candidateEvidence = body.candidates === undefined - ? [] + // Match execution: fill the same candidate evidence the router would + // assemble, so dry-run reports the same eligibility as real routing + // instead of treating every capability as unknown. + ? resolvedProfile.candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + health: policyCandidateHealthEvidence(config, candidate), + })) : parseCandidateEvidence(body.candidates); if (candidateEvidence === null) { return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config); diff --git a/tests/health-scoring.test.ts b/tests/health-scoring.test.ts new file mode 100644 index 000000000..7e0249649 --- /dev/null +++ b/tests/health-scoring.test.ts @@ -0,0 +1,326 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendUsageEntry, resetUsageReadCacheForTests, type PersistedUsageEntry } from "../src/usage/log"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; +import { + clearHealthHistoryCacheForTests, + codexPoolHealthEvidence, + healthEvidenceForCandidate, + healthScore, + HEALTH_SCORE_CONSTANTS, +} from "../src/routing/health"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +function row( + requestId: string, + status: number, + durationMs: number, + overrides: Partial = {}, +): PersistedUsageEntry { + return { + requestId, + timestamp: Date.now() - 60_000, + provider: "a", + model: "m1", + status, + durationMs, + usageStatus: "reported", + ...overrides, + }; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-health-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + clearHealthHistoryCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + }, + routingProfiles: { + healthy: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + ...overrides, + }; +} + +describe("health-aware scoring (RI-06)", () => { + test("historical evidence derives success rate, consecutive failures, latency, samples", async () => { + for (let index = 0; index < 23; index++) { + appendUsageEntry(row(`ok-${index}`, 200, 1000 + index)); + } + appendUsageEntry(row("fail-1", 503, 4000, { timestamp: Date.now() - 5_000 })); + appendUsageEntry(row("fail-2", 503, 4000, { timestamp: Date.now() - 4_000 })); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(25); + expect(evidence.failures).toBe(2); + expect(evidence.successRate).toBeCloseTo(23 / 25, 2); + expect(evidence.recentLatencyMs).toBeDefined(); + expect(evidence.incompleteStreamRate).toBeUndefined(); + const score = healthScore(evidence)!; + expect(score).toBeGreaterThan(0.5); + }); + + test("client cancellations and invalid requests never damage health", async () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 1000)); + appendUsageEntry(row("cancel", 499, 500, { closeReason: "client_cancel" })); + appendUsageEntry(row("invalid", 400, 500, { closeReason: "non_stream" })); + appendUsageEntry(row("refused", 404, 500)); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(5); + expect(evidence.successRate).toBe(1); + }); + + test("incomplete streams lower the health score", async () => { + for (let index = 0; index < 10; index++) appendUsageEntry(row(`ok-${index}`, 200, 1000)); + appendUsageEntry(row("inc", 200, 1000, { terminalStatus: "incomplete" })); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.incompleteStreamRate).toBeCloseTo(1 / 11, 2); + const score = healthScore(evidence)!; + expect(score).toBeLessThan(0.99); + }); + + test("low sample counts reduce confidence", async () => { + appendUsageEntry(row("only", 200, 1000)); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const score = healthScore(evidence)!; + const full = healthScore({ ...evidence, sampleCount: HEALTH_SCORE_CONSTANTS.MIN_CONFIDENCE_SAMPLES })!; + expect(score).toBeLessThan(full); + }); + + test("hard cooldown is authoritative: score 0 and evaluator exclusion", async () => { + const { clearCodexUpstreamHealth } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const evidence = { + sampleCount: 50, + successRate: 0.95, + cooldownUntilMs: now + 60_000, + }; + expect(healthScore(evidence, now)).toBe(0); + + const result = evaluatePolicyProfile(config(), "healthy", {}, [ + { provider: "a", model: "m1", health: { sampleCount: 50, successRate: 0.95, cooldownUntilMs: now + 60_000 } }, + { provider: "b", model: "m2", health: { sampleCount: 50, successRate: 0.95 } }, + ]); + expect(result.candidates[0]!.eligible).toBe(false); + expect(result.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cooldown")).toBe(true); + expect(result.selectedIndex).toBe(1); + }); + + test("unknown health follows the profile unknownEvidence policy", async () => { + const strict = config({ + routingProfiles: { + h: { + candidates: [{ provider: "a", model: "m1" }], + unknownEvidence: { capability: "allow", health: "exclude", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const excluded = evaluatePolicyProfile(strict, "h", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-health")).toBe(true); + expect(excluded.selectedIndex).toBeNull(); + + const penalizing = config({ + routingProfiles: { + h: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const penalized = evaluatePolicyProfile(penalizing, "h", {}, []); + // Both candidates have unknown health; penalize keeps them eligible with + // the penalized health floor folded into the score. + expect(penalized.candidates.every(candidate => candidate.eligible)).toBe(true); + expect(penalized.candidates[0]!.score!.components.health).toBe(0.3); + }); + + test("known health evidence feeds the score component and trace", async () => { + for (let index = 0; index < 30; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + const healthA = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const healthB = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + const result = evaluatePolicyProfile(config(), "healthy", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, health: healthA }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 }, health: healthB }, + ]); + const a = result.candidates[0]!; + expect(a.score!.components.health).toBeGreaterThan(0); + expect(a.score!.components.configuredPriority).toBe(1); + // Health evidence reaches the trace candidate. + expect(result.trace.candidates[0]!.health).toBeDefined(); + }); + + test("historical health moves selection between two eligible candidates", async () => { + // "a" has a bad recent streak; "b" is clean. + for (let index = 0; index < 20; index++) appendUsageEntry(row(`afail-${index}`, 503, 4000)); + for (let index = 0; index < 20; index++) { + appendUsageEntry({ + requestId: `bok-${index}`, + timestamp: Date.now() - 60_000, + provider: "b", + model: "m2", + status: 200, + durationMs: 900, + usageStatus: "reported", + }); + } + const healthA = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + const healthB = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + const healthDominant = config({ + routingProfiles: { + healthy: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + optimize: { health: 0.9 }, + }, + }, + }); + const result = evaluatePolicyProfile(healthDominant, "healthy", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, health: healthA }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 }, health: healthB }, + ]); + // With equal priority weights, the healthier candidate wins. + expect(result.selectedIndex).toBe(1); + }); + + test("execution path includes health evidence and can exclude on cooldown", async () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + const route = routeModel(config(), "policy/healthy"); + expect(route.routeKind).toBe("policy"); + expect(route.routeDecision!.candidates[0]!.health).toBeDefined(); + }); + + test("combo attempt failures contribute samples to the failed target", async () => { + for (let index = 0; index < 10; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + // Combo request: a/m1 failed as the non-final attempt, b/m2 succeeded. + appendUsageEntry({ + ...row("combo-1", 200, 1500, { provider: "b", model: "m2", timestamp: Date.now() - 1_000 }), + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 4000, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + { ordinal: 2, provider: "b", model: "m2", adapter: "openai-chat", status: 200, durationMs: 1500, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + }); + const failedTarget = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(failedTarget.sampleCount).toBe(11); + expect(failedTarget.failures).toBe(1); + const finalTarget = healthEvidenceForCandidate({ provider: "b", model: "m2" }); + expect(finalTarget.sampleCount).toBe(1); + expect(finalTarget.failures).toBeUndefined(); + expect(finalTarget.successRate).toBe(1); + }); + + test("execution path applies live codex account cooldown to openai candidates", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const cfg = config({ + providers: { + ...config().providers, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + mixed: { + candidates: [ + { provider: "openai", model: "gpt-5.6" }, + { provider: "b", model: "m2" }, + ], + }, + }, + }); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { retryAfter: "3600", now }); + const route = routeModel(cfg, "policy/mixed"); + expect(route.routeDecision!.candidates[0]!.health?.cooldownUntilMs).toBeDefined(); + expect(route.routeDecision!.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cooldown")).toBe(true); + expect(route.providerName).toBe("b"); + expect(route.modelId).toBe("m2"); + }); + + test("mixed pool cooldown/soft-avoid states degrade to soft-avoid", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const cfg = config({ + codexAccounts: [ + { id: "pool-a", email: "pool-a@example.test", isMain: false }, + { id: "pool-b", email: "pool-b@example.test", isMain: false }, + ], + }); + // pool-a hard-cooled (429); pool-b soft-avoided (transient 503s). No + // single account is usable, so the aggregate must degrade to soft-avoid. + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { retryAfter: "3600", now }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now: now + 1 }); + recordCodexUpstreamOutcome(cfg, "pool-b", 503, { now: now + 2 }); + const evidence = codexPoolHealthEvidence(cfg, now + 3); + expect(evidence?.softAvoidUntilMs).toBeDefined(); + expect(evidence?.cooldownUntilMs).toBeUndefined(); + }); + + test("unknown health under allow blends neutrally instead of outranking measured health", () => { + const cfg = config({ + routingProfiles: { + ranking: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + optimize: { latency: 0, health: 0.8, cost: 0, quota: 0 }, + unknownEvidence: { capability: "allow", health: "allow", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const result = evaluatePolicyProfile(cfg, "ranking", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + { + provider: "b", + model: "m2", + capability: { contextWindow: 200000 }, + health: { sampleCount: 50, successRate: 1, recentLatencyMs: 100 }, + }, + ]); + // a (priority 1.0, unknown -> neutral 0.5) blends to 0.5; b (priority + // 0.5, near-perfect health) blends above it. Without the neutral blend the + // unknown candidate would outrank the measured one. + expect(result.selectedIndex).toBe(1); + expect(result.candidates[0]!.score!.components.health).toBe(0.5); + }); +}); diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index ef6e242f0..77570fb96 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; import { isValidProviderName } from "../src/config"; import { getRoutingProfile } from "../src/routing/profile"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import { evidenceFromBody } from "../src/routing/request-evidence"; import type { OcxConfig } from "../src/types"; @@ -18,6 +19,7 @@ beforeEach(() => { }); afterEach(() => { + closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (testDir) rmSync(testDir, { recursive: true, force: true }); @@ -45,6 +47,16 @@ function baseConfig(overrides: Partial = {}): OcxConfig { modelContextWindows: { m2: 64_000 }, modelInputModalities: { m2: ["text"] }, }, + // Non-tool-capable adapter: keeps the "tools unknown" scenario testable + // now that `openai-chat` infers tool support from the adapter. + c: { + adapter: "bare", + baseUrl: "https://c.example/v1", + apiKey: "kc", + models: ["m3"], + modelContextWindows: { m3: 128_000 }, + modelInputModalities: { m3: ["text"] }, + }, openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, }, combos: { @@ -87,7 +99,12 @@ describe("policy execution (RI-05)", () => { expect(trace.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); expect(trace.selected.provider).toBe("a"); expect(trace.selected.model).toBe("m1"); - expect(trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + // RI-06: unknown health under the default "penalize" policy folds a + // penalized health floor into the score. + expect(trace.candidates[0]!.score).toMatchObject({ + total: 0.825, + components: { configuredPriority: 1, health: 0.3 }, + }); }); test("profile alias executes the same policy", () => { @@ -118,11 +135,11 @@ describe("policy execution (RI-05)", () => { }); test("unknown capability follows the profile unknownEvidence (exclude default)", () => { - // Provider "b" has no parallelToolCalls and no catalog row: tools unknown. + // Provider "c" uses a non-tool-capable adapter and no catalog row: tools unknown. const config = baseConfig({ routingProfiles: { toolsOnly: { - candidates: [{ provider: "b", model: "m2" }], + candidates: [{ provider: "c", model: "m3" }], require: { tools: true }, }, }, @@ -132,15 +149,45 @@ describe("policy execution (RI-05)", () => { const permissive = baseConfig({ routingProfiles: { toolsOnly: { - candidates: [{ provider: "b", model: "m2" }], + candidates: [{ provider: "c", model: "m3" }], require: { tools: true }, unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, }, }, }); const route = routeModel(permissive, "policy/toolsOnly"); + expect(route.providerName).toBe("c"); + expect(route.modelId).toBe("m3"); + }); + + test("openai-chat without the parallel-call opt-in still infers tool support", () => { + // Provider "b" is `openai-chat` with no `parallelToolCalls` and no catalog + // row: the adapter protocol itself is the tool-capability signal. + const config = baseConfig({ + routingProfiles: { + tools: { candidates: [{ provider: "b", model: "m2" }], require: { tools: true } }, + }, + }); + const route = routeModel(config, "policy/tools"); expect(route.providerName).toBe("b"); expect(route.modelId).toBe("m2"); + expect(route.routeDecision!.candidates[0]!.capability?.tools).toBe(true); + }); + + test("kiro and mimo-free adapters infer tool support", () => { + const config = baseConfig({ + providers: { + ...baseConfig().providers, + k: { adapter: "kiro", baseUrl: "https://k.example/v1", apiKey: "kk", models: ["m9"] }, + m: { adapter: "mimo-free", baseUrl: "https://m.example/v1", apiKey: "km", models: ["m8"] }, + }, + routingProfiles: { + ktools: { candidates: [{ provider: "k", model: "m9" }], require: { tools: true } }, + mtools: { candidates: [{ provider: "m", model: "m8" }], require: { tools: true } }, + }, + }); + expect(routeModel(config, "policy/ktools")).toMatchObject({ providerName: "k", modelId: "m9" }); + expect(routeModel(config, "policy/mtools")).toMatchObject({ providerName: "m", modelId: "m8" }); }); test("request evidence constrains candidates: image input excludes non-image models", () => { @@ -159,14 +206,23 @@ describe("policy execution (RI-05)", () => { test("request tools requirement is enforced when provably needed", () => { const config = baseConfig({ routingProfiles: { - tools: { candidates: [{ provider: "b", model: "m2" }] }, + tools: { candidates: [{ provider: "c", model: "m3" }] }, }, }); - expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "b", modelId: "m2" }); - // b's tools support is unknown -> request requiring tools excludes it. + expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "c", modelId: "m3" }); + // c's tools support is unknown -> request requiring tools excludes it. expect(() => routeModel(config, "policy/tools", { toolsRequired: true })).toThrow(NoEligiblePolicyCandidateError); }); + test("unresolved policy/ falls through to normal resolution", () => { + const config = baseConfig(); + // No profile named "nope": the reserved-looking id must not throw and not + // shadow provider/default resolution. + const route = routeModel(config, "policy/nope"); + expect(route.routeKind).toBe("default-provider"); + expect(route.providerName).toBe("a"); + }); + test("policy selection is deterministic across calls", () => { const config = baseConfig(); const first = routeModel(config, "policy/fast"); diff --git a/tests/request-history-index.test.ts b/tests/request-history-index.test.ts index a1e3573c6..64d2abc1b 100644 --- a/tests/request-history-index.test.ts +++ b/tests/request-history-index.test.ts @@ -109,6 +109,23 @@ describe("request-history index (RI-02)", () => { expect(after.meta.lastError).not.toMatch(/identity changed/i); }); + test("appended rows are ingested as a tail, never a full rebuild", async () => { + for (const row of seedRows(5)) appendUsageEntry(row); + const first = await queryRequestHistory({}, undefined, 10); + expect(first.meta.indexedRows).toBe(5); + // A subsequent append is ingested from the indexed offset and clears the + // rebuild marker (routing-time health reads depend on this not re-parsing + // the whole ledger synchronously). + appendUsageEntry(entry("late-1", 7000)); + const second = await queryRequestHistory({}, undefined, 10); + expect(second.meta.indexedRows).toBe(6); + expect(second.meta.lastError).toBe(""); + appendUsageEntry(entry("late-2", 8000)); + const third = await queryRequestHistory({}, undefined, 10); + expect(third.meta.indexedRows).toBe(7); + expect(third.meta.lastError).toBe(""); + }); + test("large history indexes fully and paginates without duplicates or misses", async () => { const rows = seedRows(1500, 10_000); for (const row of rows) appendUsageEntry(row); diff --git a/tests/route-decision-trace.test.ts b/tests/route-decision-trace.test.ts index cb6fd6a59..5a0c00579 100644 --- a/tests/route-decision-trace.test.ts +++ b/tests/route-decision-trace.test.ts @@ -274,6 +274,34 @@ describe("route decision traces (RI-01)", () => { expect(trace.truncated?.strings).toBe(true); }); + test("caller-supplied evidence is whitelisted and bounded in the trace", () => { + const trace = buildRouteDecisionTrace({ + requestedModel: "policy/p", + routeKind: "policy", + candidates: [{ + provider: "a", + model: "m1", + eligible: true, + exclusions: [], + capability: { + serviceTier: "x".repeat(500), + reasoningEfforts: ["y".repeat(500)], + unknownNested: { z: "should-not-survive" }, + }, + quota: { known: true, source: "s".repeat(300) }, + cost: { estimatedUsd: 0.5, priceSource: "p".repeat(300) }, + }], + selected: { provider: "a", model: "m1", reason: "policy-selected" }, + }); + const candidate = trace.candidates[0]!; + expect(candidate.capability?.serviceTier?.length).toBe(MAX_TRACE_STRING); + expect(candidate.capability?.reasoningEfforts?.[0]?.length).toBe(MAX_TRACE_STRING); + expect(candidate.capability && "unknownNested" in candidate.capability).toBe(false); + expect(candidate.quota?.source?.length).toBe(MAX_TRACE_STRING); + expect(candidate.cost?.priceSource?.length).toBe(MAX_TRACE_STRING); + expect(trace.truncated?.strings).toBe(true); + }); + test("trace never contains credentials or prompt content", () => { const config = baseConfig(); // Built at runtime so the privacy scanner's key-pattern grep does not diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index 4865e5687..c7ddc9e02 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { validateConfigCandidate } from "../src/config"; import { handleManagementAPI } from "../src/server/management-api"; import { ManagementRequest } from "./helpers/management-auth"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import { getRoutingProfile, listRoutingProfileIds, @@ -27,6 +28,7 @@ beforeEach(() => { }); afterEach(() => { + closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (testDir) rmSync(testDir, { recursive: true, force: true }); @@ -154,7 +156,13 @@ describe("routing profiles (RI-04)", () => { candidates: [{ provider: "a", model: "m1" }], alias: "a/m1", }, config); - expect(providerNamespaceCollision.some(issue => issue.message.includes("provider routing namespace"))).toBe(true); + // Exactly one issue: the first-segment provider collision must not be + // reported twice with different wordings. + const namespaceIssues = providerNamespaceCollision.filter( + issue => issue.message.includes("provider routing namespace"), + ); + expect(namespaceIssues.length).toBe(1); + expect(providerNamespaceCollision.length).toBe(1); const siblingCollision = routingProfileIssues("p", { candidates: [{ provider: "a", model: "m1" }], @@ -307,7 +315,12 @@ describe("routing profiles (RI-04)", () => { { provider: "b", model: "m2", capability: { contextWindow: 5000 } }, ]); expect(result.selectedIndex).toBe(0); - expect(result.trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + // RI-06: unknown health under the default "penalize" policy folds a + // penalized health floor into the score. + expect(result.trace.candidates[0]!.score).toMatchObject({ + total: 0.825, + components: { configuredPriority: 1, health: 0.3 }, + }); }); test("API lists profiles and dry-runs deterministically", async () => { @@ -361,4 +374,66 @@ describe("routing profiles (RI-04)", () => { const badResponse = await handleManagementAPI(badReq, new URL(badReq.url), config, { refreshCodexCatalog: async () => {} }); expect(badResponse!.status).toBe(400); }); + + test("API dry-run without explicit candidates fills the same evidence as execution", async () => { + const config = baseConfig({ + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"], modelContextWindows: { m1: 200_000 }, parallelToolCalls: true }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"], modelContextWindows: { m2: 64_000 } }, + }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + }, + }, + }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { selectedIndex?: number | null; candidates?: Array<{ provider?: string; eligible?: boolean }> }; + // a: 200k context + openai-chat tools => eligible; b: 64k below the hard + // minimum => excluded, exactly like real routing would report. + expect(body.selectedIndex).toBe(0); + expect(body.candidates?.[0]).toMatchObject({ provider: "a", eligible: true }); + expect(body.candidates?.[1]).toMatchObject({ provider: "b", eligible: false }); + }); + + test("API dry-run mirrors live codex cooldown for openai candidates", async () => { + const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../src/codex/routing"); + clearCodexUpstreamHealth(); + const now = Date.now(); + const config = baseConfig({ + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"], modelContextWindows: { m1: 200_000 }, parallelToolCalls: true }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + only: { candidates: [{ provider: "openai", model: "gpt-5.6" }] }, + }, + }); + recordCodexUpstreamOutcome(config, "pool-a", 429, { retryAfter: "3600", now }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "only", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { candidates?: Array<{ eligible?: boolean; exclusions?: Array<{ code: string }> }> }; + expect(body.candidates?.[0]?.eligible).toBe(false); + expect(body.candidates?.[0]?.exclusions?.some(exclusion => exclusion.code === "cooldown")).toBe(true); + }); });