diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 3e32843b57..ed6eff6963 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -10,7 +10,8 @@ head SHA, PR number/URL, verification result, and review state. - Bun: `1.3.14`; package version: `2.10.0` - Worktree: `D:\codex-worktrees\ocx-router-intelligence` - Push remote: `origin` (Wibias/opencodex); PR target: `lidge-jun/opencodex:dev` -- Programme stack: #1003 (RI-01) and #1004 (RI-02) merged to `dev`; #1005 (RI-03) open. +- Programme stack: #1003 (RI-01), #1004 (RI-02), and #1005 (RI-03) merged to + `dev`; #1011 (RI-04) open. ## Related in-flight PRs (not superseded by this stack) @@ -41,8 +42,8 @@ other; closing one is a maintainer decision and neither is stale. |---|---|---|---|---|---|---| | RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | MERGED | | RI-02 | `feat/ri-02-request-history-index` | `dev` (post-#1003 merge) | `2a72aa4a9` | #1004 | https://github.com/lidge-jun/opencodex/pull/1004 | MERGED | -| RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | pending | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | OPEN (resync) | -| RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | +| RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | `a594938c5` | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | MERGED | +| RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | OPEN (resync) | | RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | | RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | | RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | @@ -131,8 +132,9 @@ other; closing one is a maintainer decision and neither is stale. (2) SQL column names are snake_case - analytics SELECT now aliases to camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not `costUsd`; plus the row-cap is injectable for truncation tests. -- Final commit: `5f464c730` (CodeRabbit: cooldown parse gate, API `limit` default 5k, devlog + tests) -- PR: #1005 (OPEN) https://github.com/lidge-jun/opencodex/pull/1005 +- Final commit: `5f464c730` (CodeRabbit: cooldown parse gate, API `limit` default 5k, devlog + tests); + merged head on `dev`: `a594938c5` +- PR: #1005 (MERGED) https://github.com/lidge-jun/opencodex/pull/1005 - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) - `bun run test tests/routing-analytics.test.ts`: 10/10 pass: @@ -143,3 +145,37 @@ other; closing one is a maintainer decision and neither is stale. - Focused regression suites: 144/144 pass across 6 files - `bun run privacy:scan`: passed - Remaining Low findings: none + +### RI-04 - feat/ri-04-policy-profile-core + +- Base SHA: `a594938c5` (`dev` after #1005 merge; rebased from the RI-03 head + when #1003/#1004/#1005 landed) +- Reviewed commits: `63924495e` (RI-04 core) + review round `d478b393` + (request-evidence wiring), `8e1f1c3d` (provider-namespace alias check, dead + export removal), `aa9212fa` (CLI cleanup), `27511bf7` (absent-flag + semantics, cost-limit enforcement, deterministic id ordering, CodeRabbit + round) +- Findings (self-review): 4 fixed pre-push - (1) `serviceTier` evidence type + was `Unknownable` (number|boolean) but service tiers are strings - trace + type narrowed to `string | "unknown"`; (2) alias validation missed the + reserved `combo/` namespace prefix; (3) trace candidates did not carry + `score` - added `score` to `TraceCandidateInput`/`buildCandidate`; + (4) test expectation for weight normalization used wrong math (unspecified + weights keep defaults; sum 4.35 not 4). +- Final commit: `27511bf7` (see Reviewed commits) +- PR: #1011 (OPEN, ready) https://github.com/lidge-jun/opencodex/pull/1011 +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/routing-profile.test.ts`: 14/14 pass (validation, + normalization, revision digest, collisions incl. provider namespace, + config load, id/alias resolution, dry-run eligibility incl. request + evidence and cost limit, unknown/tie-break, API list+dry-run, + API error codes) + - Focused regression suites: pass across route-decision-trace, + routing-analytics, request-history-index, combos, codex-routing, + internal-cli-dispatch + - `bun run privacy:scan`: passed + - `tests/config.test.ts`: 109/115 pass; the 6 symlink failures reproduce + identically on the pristine base (Windows symlink EPERM, environmental) +- Remaining Low findings: none (B3/B5 residual: no-eligible trace names + candidate 0 as `selected`; API evidence fields are permissively dropped) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 85a5c89176..1cf9781c08 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -11,6 +11,7 @@ Routing turns the model id sent by a client into one concrete provider and upstr | --- | --- | --- | --- | | `defaultProvider` | `string` | `"openai"` | Final provider used when no earlier model rule matches. It must name an enabled configured provider. | | `combos?` | `Record` | `{}` | Virtual `combo/` models built from ordered provider/model targets. | +| `routingProfiles?` | `Record` | `{}` | Virtual `policy/` models that select among an explicit candidate allowlist using hard capability requirements and deterministic scoring. | ## Model resolution order @@ -82,6 +83,78 @@ namespace, and cannot use reserved bare native families such as `gpt-*`, `o1-*`, For strategy behavior, retryable failures, cooldowns, encrypted v2 task limits, and management commands, see [Combos](/guides/combos/). +## Routing policy profiles (`config.routingProfiles`) + +Routing policy profiles are the Router Intelligence selection layer: an explicitly requested +`policy/` (or configured alias) selects among a fixed candidate allowlist using hard capability +requirements and deterministic, explainable scoring. In this release profiles are configuration and +dry-run evaluation only: production requests are not yet routed through them (execution wiring +arrives with RI-05), and existing model ids are **never** routed through a profile implicitly. +Policy ids do not participate in the model resolution order above until execution lands. + +Each key is an id matching `[A-Za-z0-9][A-Za-z0-9._-]{0,63}`, always addressable as `policy/`, +with one optional `alias`. Aliases must be unique and cannot collide with configured providers, +the `/` routing namespace, combos, codex account namespaces, the `policy/` +namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `codex-*`). + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `candidates` | `{ provider: string; model: string }[]` | required | Explicit allowlist of `provider/model` refs. No implicit expansion. | +| `alias?` | `string` | — | Optional public model id in place of `policy/`. | +| `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | +| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights; normalized deterministically. | +| `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | +| `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | + +`require` supports: `minContextWindow` (positive integer), and the booleans `tools`, `imageInput`, +`structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`; plus `reasoningEffort` and +`serviceTier` strings. + +Request evidence supplied to a dry-run (context window, tools, image input, structured output, +reasoning effort, service tier, encrypted Codex tasks) is evaluated against candidate capabilities +together with the profile `require` block; a candidate must satisfy both to be eligible. + +The CLI dry-run accepts request-evidence flags but cannot supply candidate capability evidence yet; +candidate evidence is provided through the API (`POST /api/routing-profiles/dry-run`). + +```json +{ + "routingProfiles": { + "fast": { + "alias": "ocx/fast", + "candidates": [ + { "provider": "anthropic", "model": "claude-sonnet-5" }, + { "provider": "openai", "model": "gpt-5.6-sol" } + ], + "require": { "tools": true, "minContextWindow": 128000 }, + "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, + "limits": { "maxEstimatedCostUsd": 0.50 }, + "unknownEvidence": { + "capability": "exclude", + "health": "penalize", + "quota": "penalize", + "cost": "penalize" + } + } + } +} +``` + +CLI: `ocx route policy list`, `ocx route policy show `, and +`ocx route policy dry-run --model-context --tools`. Dry-run evaluates candidates +without sending any upstream request. + +### Combos vs policy profiles + +- A **combo** is explicit ordered/weighted target routing and failover: the configured order (or + smooth weighted round-robin) decides, and failures advance through the list. +- A **policy profile** is evidence-based selection among configured candidates: hard capability + requirements filter first, then deterministic scoring ranks the survivors. + +Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate +is chosen. Profile scoring will expand with capability (RI-05), health (RI-06), quota (RI-07), and +cost (RI-08) dimensions; per-request trace recording arrives with execution (RI-05). + ### Catalog eligibility A combo remains directly routable even when it cannot be listed. `ocx sync`, `/v1/models`, and the diff --git a/src/cli/index.ts b/src/cli/index.ts index 8557613596..afaa440934 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1007,13 +1007,18 @@ switch (command) { break; } case "route": { - if (args[1] !== "combo") { - console.error("Usage: ocx route combo "); + if (args[1] !== "combo" && args[1] !== "policy") { + console.error("Usage: ocx route "); process.exitCode = 2; break; } - const { handleComboCommand } = await import("./combo"); - process.exitCode = await handleComboCommand(args.slice(2)); + if (args[1] === "combo") { + const { handleComboCommand } = await import("./combo"); + process.exitCode = await handleComboCommand(args.slice(2)); + } else { + const { handleRoutePolicyCommand } = await import("./route-policy"); + process.exitCode = await handleRoutePolicyCommand(args.slice(2)); + } break; } case "agent": { diff --git a/src/cli/route-policy.ts b/src/cli/route-policy.ts new file mode 100644 index 0000000000..eb58552e74 --- /dev/null +++ b/src/cli/route-policy.ts @@ -0,0 +1,90 @@ +import { + CliUsageError, + printData, + rejectArgs, + runCliAction, + runtimeRequest, + takeFlag, + takeIntegerOption, + type RuntimeApiDeps, +} from "./runtime-api"; + +const USAGE = `Usage: + ocx route policy list [--json] + ocx route policy show [--json] + ocx route policy dry-run [--model-context ] [--tools] + [--image] [--structured-output] [--json]`; + +interface ProfileRow { + id?: string; + model?: string; + revision?: string; +} + +async function list(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + rejectArgs(args, USAGE); + const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps); + const rows = result.profiles ?? []; + printData( + result, + wantsJson, + rows.length + ? rows.map(row => `${String(row.id)} ${String(row.model ?? `policy/${row.id}`)} rev:${String(row.revision ?? "-")}`) + : ["No routing profiles configured."], + ); +} + +async function show(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift(); + const wantsJson = takeFlag(args, "--json"); + if (!id) throw new CliUsageError("profile id is required", USAGE); + rejectArgs(args, USAGE); + const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps); + const profile = (result.profiles ?? []).find(candidate => candidate.id === id); + if (!profile) throw new CliUsageError(`unknown routing profile: ${id}`, USAGE); + printData(profile, wantsJson); +} + +async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift(); + const wantsJson = takeFlag(args, "--json"); + if (!id) throw new CliUsageError("profile id is required", USAGE); + const modelContext = takeIntegerOption(args, "--model-context", { min: 1 }); + const tools = takeFlag(args, "--tools"); + const image = takeFlag(args, "--image"); + const structuredOutput = takeFlag(args, "--structured-output"); + rejectArgs(args, USAGE); + const result = await runtimeRequest( + "/api/routing-profiles/dry-run", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: id, + evidence: { + ...(modelContext !== undefined ? { contextWindow: modelContext } : {}), + ...(tools ? { toolsRequired: true } : {}), + ...(image ? { imageInputRequired: true } : {}), + ...(structuredOutput ? { structuredOutputRequired: true } : {}), + }, + }), + }, + deps, + ); + printData(result, wantsJson); +} + +export async function handleRoutePolicyCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise { + return runCliAction(async () => { + const [sub, ...rest] = argv; + if (!sub) throw new CliUsageError("route policy requires a subcommand (list, show, dry-run)", USAGE); + if (sub === "list") await list(rest, deps); + else if (sub === "show") await show(rest, deps); + else if (sub === "dry-run") await dryRun(rest, deps); + else throw new CliUsageError(`unknown route policy command: ${sub}`, USAGE); + }); +} diff --git a/src/config.ts b/src/config.ts index 72defeb9ca..b9b9d7c09f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,7 @@ import { MAIN_CODEX_ACCOUNT_NAMESPACE_TARGET, } from "./codex/account-namespace-match"; import { COMBO_NAMESPACE, comboConfigIssues } from "./combos/types"; +import { routingProfileIssues } from "./routing/profile"; import { forgetEphemeralSecretPath, hardenSecretDir, @@ -1261,6 +1262,27 @@ const configSchema = z.object({ } } } + const routingProfiles = (config as { routingProfiles?: unknown }).routingProfiles; + if (routingProfiles !== undefined) { + if (!routingProfiles || typeof routingProfiles !== "object" || Array.isArray(routingProfiles)) { + ctx.addIssue({ code: "custom", path: ["routingProfiles"], message: "routingProfiles must be an object" }); + } else { + for (const [id, raw] of Object.entries(routingProfiles as Record)) { + for (const issue of routingProfileIssues(id, raw, { + providers: config.providers, + combos: combos as Record | undefined, + routingProfiles: routingProfiles as Record, + codexAccountNamespaces: accountNamespaces, + }, { excludeProfileId: id })) { + ctx.addIssue({ + code: "custom", + path: ["routingProfiles", id, ...issue.path], + message: issue.message, + }); + } + } + } + } }); /** diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts new file mode 100644 index 0000000000..880403e2e7 --- /dev/null +++ b/src/routing/evaluator.ts @@ -0,0 +1,340 @@ +/** + * Deterministic policy-profile evaluator (RI-04 core; extended by RI-05..08). + * + * RI-04 evaluates hard capability requirements against supplied evidence and + * scores by deterministic configured priority only. It never dispatches an + * upstream request; execution wiring arrives with RI-05. + */ + +import type { OcxConfig } from "../types"; +import { + buildRouteDecisionTrace, + type RouteCapabilityEvidence, + type RouteCostEvidence, + type RouteDecisionTraceV1, + type RouteExclusionReason, + type RouteHealthEvidence, + type RouteQuotaEvidence, + type RouteRequirementEvidence, + type RouteScoreEvidence, + type Unknownable, +} from "./trace"; +import { getRoutingProfile, policyModelId, type NormalizedRoutingProfile } from "./profile"; + +export interface PolicyRequestEvidence { + /** Required context window for this request (tokens). */ + contextWindow?: number; + toolsRequired?: boolean; + imageInputRequired?: boolean; + structuredOutputRequired?: boolean; + reasoningEffort?: string; + serviceTier?: string; + encryptedCodexTask?: boolean; +} + +export interface PolicyCandidateEvidence { + provider: string; + model: string; + accountRef?: string; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; +} + +export interface PolicyEvaluationCandidate { + provider: string; + model: string; + accountRef?: string; + eligible: boolean; + exclusions: RouteExclusionReason[]; + requirements: RouteRequirementEvidence[]; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; + score?: RouteScoreEvidence; +} + +export interface PolicyEvaluationResult { + profileId: string; + profileRevision: string; + candidates: PolicyEvaluationCandidate[]; + selectedIndex: number | null; + trace: RouteDecisionTraceV1; +} + +function booleanRequirement( + id: string, + required: boolean | undefined, + actual: Unknownable | undefined, +): RouteRequirementEvidence | null { + if (required === undefined) return null; + // Note: `required === false` is a negative assertion ("must NOT have X"), + // not "don't care"; an absent field is the only "no requirement" form. + if (actual === undefined || actual === "unknown") { + return { id, expected: required, outcome: "unknown" }; + } + return { + id, + expected: required, + actual: typeof actual === "boolean" ? actual : String(actual), + outcome: typeof actual === "boolean" && actual === required ? "satisfied" : "unsatisfied", + }; +} + +function requirementFor( + require: NormalizedRoutingProfile["require"], + capability: RouteCapabilityEvidence | undefined, +): RouteRequirementEvidence[] { + const requirements: RouteRequirementEvidence[] = []; + if (require.minContextWindow !== undefined) { + const actual = capability?.contextWindow; + if (typeof actual === "number") { + requirements.push({ + id: "min-context-window", + expected: require.minContextWindow, + actual, + outcome: actual >= require.minContextWindow ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "min-context-window", expected: require.minContextWindow, outcome: "unknown" }); + } + } + const tools = booleanRequirement("tools", require.tools, capability?.tools); + if (tools) requirements.push(tools); + const image = booleanRequirement("image-input", require.imageInput, capability?.image); + if (image) requirements.push(image); + const structured = booleanRequirement("structured-output", require.structuredOutput, capability?.structuredOutput); + if (structured) requirements.push(structured); + if (require.reasoningEffort !== undefined) { + const ladder = capability?.reasoningEfforts; + if (Array.isArray(ladder)) { + requirements.push({ + id: "reasoning-effort", + expected: require.reasoningEffort, + actual: ladder.join(","), + outcome: ladder.includes(require.reasoningEffort) ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "reasoning-effort", expected: require.reasoningEffort, outcome: "unknown" }); + } + } + if (require.serviceTier !== undefined) { + const actual = capability?.serviceTier; + if (actual === undefined || actual === "unknown") { + requirements.push({ id: "service-tier", expected: require.serviceTier, outcome: "unknown" }); + } else { + requirements.push({ + id: "service-tier", + expected: require.serviceTier, + actual: String(actual), + outcome: actual === require.serviceTier ? "satisfied" : "unsatisfied", + }); + } + } + const local = booleanRequirement("local-only", require.localOnly, capability?.localOnly); + if (local) requirements.push(local); + const remote = booleanRequirement("remote-allowed", require.remoteAllowed, capability?.remoteAllowed); + if (remote) requirements.push(remote); + const encrypted = booleanRequirement( + "encrypted-codex-tasks", + require.encryptedCodexTasks, + capability?.encryptedCodexTasks, + ); + if (encrypted) requirements.push(encrypted); + return requirements; +} + +function unsatisfiedOrUnknown(requirements: RouteRequirementEvidence[]): RouteRequirementEvidence[] { + return requirements.filter(requirement => requirement.outcome !== "satisfied"); +} + +function configuredPriorityScore(index: number, total: number): number { + return total > 1 ? (total - index) / total : 1; +} + +/** + * Requirements derived from the request evidence supplied to a dry-run. A + * candidate must satisfy both the profile `require` block and the request + * needs (context window, tools, image input, structured output, reasoning + * effort, service tier, encrypted Codex tasks) to be eligible. + */ +function requestRequirementFor( + request: PolicyRequestEvidence, + capability: RouteCapabilityEvidence | undefined, +): RouteRequirementEvidence[] { + const requirements: RouteRequirementEvidence[] = []; + if (request.contextWindow !== undefined) { + const actual = capability?.contextWindow; + if (typeof actual === "number") { + requirements.push({ + id: "request-context-window", + expected: request.contextWindow, + actual, + outcome: actual >= request.contextWindow ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "request-context-window", expected: request.contextWindow, outcome: "unknown" }); + } + } + // Absent flags mean "no requirement": only a positive request need adds a row. + if (request.toolsRequired === true) { + const tools = booleanRequirement("request-tools", true, capability?.tools); + if (tools) requirements.push(tools); + } + if (request.imageInputRequired === true) { + const image = booleanRequirement("request-image-input", true, capability?.image); + if (image) requirements.push(image); + } + if (request.structuredOutputRequired === true) { + const structured = booleanRequirement("request-structured-output", true, capability?.structuredOutput); + if (structured) requirements.push(structured); + } + if (request.reasoningEffort !== undefined) { + const ladder = capability?.reasoningEfforts; + if (Array.isArray(ladder)) { + requirements.push({ + id: "request-reasoning-effort", + expected: request.reasoningEffort, + actual: ladder.join(","), + outcome: ladder.includes(request.reasoningEffort) ? "satisfied" : "unsatisfied", + }); + } else { + requirements.push({ id: "request-reasoning-effort", expected: request.reasoningEffort, outcome: "unknown" }); + } + } + if (request.serviceTier !== undefined) { + const actual = capability?.serviceTier; + if (actual === undefined || actual === "unknown") { + requirements.push({ id: "request-service-tier", expected: request.serviceTier, outcome: "unknown" }); + } else { + requirements.push({ + id: "request-service-tier", + expected: request.serviceTier, + actual: String(actual), + outcome: actual === request.serviceTier ? "satisfied" : "unsatisfied", + }); + } + } + if (request.encryptedCodexTask === true) { + const encrypted = booleanRequirement( + "request-encrypted-codex-tasks", + true, + capability?.encryptedCodexTasks, + ); + if (encrypted) requirements.push(encrypted); + } + return requirements; +} + +/** + * Evaluate a profile against request + candidate evidence. Deterministic: + * candidates are scored in declaration order, ties break by earlier index. + */ +export function evaluatePolicyProfile( + config: OcxConfig, + profileId: string, + requestEvidence: PolicyRequestEvidence, + candidateEvidence: PolicyCandidateEvidence[], +): PolicyEvaluationResult { + const profile = getRoutingProfile(config, profileId); + if (!profile) throw new Error(`Unknown routing profile: ${profileId}`); + + const candidates: PolicyEvaluationCandidate[] = []; + let selectedIndex: number | null = null; + let bestScore = Number.NEGATIVE_INFINITY; + + profile.candidates.forEach((declared, index) => { + const evidence = candidateEvidence.find( + candidate => candidate.provider === declared.provider && candidate.model === declared.model, + ) ?? { provider: declared.provider, model: declared.model }; + const requirements = [ + ...requirementFor(profile.require, evidence.capability), + ...requestRequirementFor(requestEvidence, evidence.capability), + ]; + const exclusions: RouteExclusionReason[] = []; + const bad = unsatisfiedOrUnknown(requirements); + for (const requirement of bad) { + if (requirement.outcome === "unsatisfied") { + exclusions.push({ code: "capability-unsatisfied", detail: requirement.id }); + } else { + exclusions.push({ code: "unknown-capability", detail: requirement.id }); + } + } + const unsatisfied = bad.some(requirement => requirement.outcome === "unsatisfied"); + const unknown = bad.some(requirement => requirement.outcome === "unknown"); + // Unknown capability handling per profile: exclude (default), penalize, + // or allow. "penalize" currently cannot move the score because RI-04 has + // no capability component yet - the capability score arrives with RI-05. + const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; + const costLimit = profile.limits.maxEstimatedCostUsd; + const estimatedCost = evidence.cost?.estimatedUsd; + const overCostLimit = costLimit !== undefined + && typeof estimatedCost === "number" + && Number.isFinite(estimatedCost) + && estimatedCost > costLimit; + if (overCostLimit) { + exclusions.push({ code: "cost-limit", detail: "maxEstimatedCostUsd" }); + } + const eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + + const priorityScore = configuredPriorityScore(index, profile.candidates.length); + const score: RouteScoreEvidence = { + total: priorityScore, + components: { configuredPriority: priorityScore }, + }; + const evaluated: PolicyEvaluationCandidate = { + provider: evidence.provider, + model: evidence.model, + ...(evidence.accountRef ? { accountRef: evidence.accountRef } : {}), + eligible, + exclusions, + requirements, + ...(evidence.capability ? { capability: evidence.capability } : {}), + ...(evidence.health ? { health: evidence.health } : {}), + ...(evidence.quota ? { quota: evidence.quota } : {}), + ...(evidence.cost ? { cost: evidence.cost } : {}), + score, + }; + candidates.push(evaluated); + + if (evaluated.eligible && score.total > bestScore) { + bestScore = score.total; + selectedIndex = index; + } + }); + + const trace = buildRouteDecisionTrace({ + requestedModel: policyModelId(profileId), + routeKind: "policy", + profile: { id: profile.id, revision: profile.revision }, + // Flat, capped summary (truncation is flagged by the builder); per-candidate + // attribution lives in each candidate's `requirements`/`exclusions`. + requirements: candidates.flatMap(candidate => candidate.requirements).slice(0, 16), + candidates: candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + ...(candidate.accountRef ? { accountRef: candidate.accountRef } : {}), + eligible: candidate.eligible, + exclusions: candidate.exclusions, + ...(candidate.score ? { score: candidate.score } : {}), + })), + selected: selectedIndex === null + ? { provider: candidates[0]?.provider ?? "", model: candidates[0]?.model ?? "", reason: "no-eligible-candidate" } + : { + candidateIndex: selectedIndex, + provider: candidates[selectedIndex]!.provider, + model: candidates[selectedIndex]!.model, + reason: "policy-selected", + }, + }); + + return { + profileId: profile.id, + profileRevision: profile.revision, + candidates, + selectedIndex, + trace, + }; +} diff --git a/src/routing/profile.ts b/src/routing/profile.ts new file mode 100644 index 0000000000..bc5ac85c39 --- /dev/null +++ b/src/routing/profile.ts @@ -0,0 +1,413 @@ +/** + * Routing policy profiles (RI-04): schema validation, normalization, revision + * digest, and id/alias resolution. Mirrors the combos module discipline + * (`src/combos/types.ts`) so both virtual-routing namespaces stay consistent. + */ + +import { createHash } from "node:crypto"; +import type { + OcxConfig, + OcxRoutingProfileConfig, + OcxRoutingUnknownEvidenceMode, +} from "../types"; +import { codexAccountNamespaceEntries } from "../codex/account-namespaces"; +import { listComboIds, resolveComboId } from "../combos"; +import { hasOwnProvider } from "../config"; + +export const POLICY_NAMESPACE = "policy"; + +export const POLICY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +export const POLICY_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}(?:\/[A-Za-z0-9][A-Za-z0-9._-]{0,63})?$/; +export const NATIVE_OPENAI_FAMILY_PATTERN = /^(?:gpt-|o1-|o3-|o4-|codex-)/; + +export const DEFAULT_PROFILE_WEIGHTS = { + latency: 0.55, + health: 0.25, + cost: 0.10, + quota: 0.10, +} as const; + +export const DEFAULT_UNKNOWN_EVIDENCE: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode> = { + capability: "exclude", + health: "penalize", + quota: "penalize", + cost: "penalize", +}; + +export interface RoutingProfileValidationIssue { + path: Array; + message: string; +} + +export interface NormalizedRoutingProfileRequirements { + minContextWindow?: number; + tools?: boolean; + imageInput?: boolean; + structuredOutput?: boolean; + reasoningEffort?: string; + serviceTier?: string; + localOnly?: boolean; + remoteAllowed?: boolean; + encryptedCodexTasks?: boolean; +} + +export interface NormalizedRoutingProfile { + id: string; + alias: string | null; + candidates: Array<{ provider: string; model: string }>; + require: NormalizedRoutingProfileRequirements; + optimize: { latency: number; health: number; cost: number; quota: number }; + limits: { maxEstimatedCostUsd?: number }; + unknownEvidence: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode>; + revision: string; +} + +const REQUIRE_KEYS = [ + "minContextWindow", + "tools", + "imageInput", + "structuredOutput", + "reasoningEffort", + "serviceTier", + "localOnly", + "remoteAllowed", + "encryptedCodexTasks", +] as const; + +const UNKNOWN_EVIDENCE_KEYS = ["capability", "health", "quota", "cost"] as const; + +export function isValidPolicyId(id: string): boolean { + return POLICY_ID_PATTERN.test(id); +} + +export function policyModelId(id: string): string { + return `${POLICY_NAMESPACE}/${id}`; +} + +export function policyPublicModelId(id: string, profile: { alias?: string | null }): string { + const alias = typeof profile.alias === "string" ? profile.alias.trim() : ""; + return alias || policyModelId(id); +} + +export function parsePolicyModelId(modelId: string): string | null { + const slash = modelId.indexOf("/"); + if (slash <= 0 || modelId.slice(0, slash) !== POLICY_NAMESPACE) return null; + const id = modelId.slice(slash + 1); + return id.length > 0 ? id : null; +} + +/** + * Resolve a client-requested model id to a policy profile id. The canonical + * `policy/` form wins first; otherwise an exact alias match. + */ +export function resolvePolicyProfileId( + config: { routingProfiles?: Record }, + modelId: string, +): string | null { + const direct = parsePolicyModelId(modelId); + if (direct) return direct; + const profiles = config.routingProfiles; + if (!profiles) return null; + for (const [id, raw] of Object.entries(profiles)) { + if (!raw || typeof raw !== "object") continue; + const alias = typeof raw.alias === "string" ? raw.alias.trim() : ""; + if (alias && alias === modelId) return id; + } + return null; +} + +function aliasIssues( + id: string, + alias: string, + config: Pick, + options: { excludeProfileId?: string } = {}, +): RoutingProfileValidationIssue[] { + const issues: RoutingProfileValidationIssue[] = []; + if (!POLICY_ALIAS_PATTERN.test(alias)) { + issues.push({ + path: ["alias"], + message: "alias must use letters, numbers, dot, underscore, or hyphen, with at most one \"/\" segment", + }); + return issues; + } + if (alias === POLICY_NAMESPACE || alias.startsWith(`${POLICY_NAMESPACE}/`)) { + issues.push({ + path: ["alias"], + message: `alias must not use the reserved "${POLICY_NAMESPACE}/" namespace`, + }); + } + if (alias === "combo" || alias.startsWith("combo/")) { + issues.push({ + path: ["alias"], + message: `alias must not use the reserved "combo/" namespace`, + }); + } + if (!alias.includes("/") && NATIVE_OPENAI_FAMILY_PATTERN.test(alias)) { + issues.push({ + path: ["alias"], + message: "bare aliases in the OpenAI native family (gpt-*, o1-*, o3-*, o4-*, codex-*) are not allowed", + }); + } + // Cross-namespace collisions: providers, combos, account namespaces, and + // sibling profile aliases all own public model ids that must stay unique. + if (hasOwnProvider(config.providers, alias)) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with configured provider name "${alias}"` }); + } + if (resolveComboId({ combos: config.combos }, alias)) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured combo selector` }); + } + if (alias.includes("/") && codexAccountNamespaceEntries(config).some(([namespace]) => namespace === alias.split("/")[0])) { + issues.push({ path: ["alias"], message: `alias "${alias}" collides with a configured codex account namespace` }); + } + if (alias.includes("/") && hasOwnProvider(config.providers, alias.split("/")[0])) { + issues.push({ + path: ["alias"], + message: `alias "${alias}" collides with the provider routing namespace "${alias.split("/")[0]}"`, + }); + } + for (const [otherId, other] of Object.entries(config.routingProfiles ?? {})) { + if (otherId === id || otherId === options.excludeProfileId) continue; + const otherAlias = typeof other?.alias === "string" ? other.alias.trim() : ""; + if (otherAlias && otherAlias === alias) { + issues.push({ path: ["alias"], message: `alias "${alias}" is already used by profile "${otherId}"` }); + } + } + return issues; +} + +export function routingProfileIssues( + id: string, + raw: unknown, + config: Pick, + options: { excludeProfileId?: string } = {}, +): RoutingProfileValidationIssue[] { + const issues: RoutingProfileValidationIssue[] = []; + if (!isValidPolicyId(id)) { + issues.push({ + path: [], + message: "profile id must start with a letter/number and use letters, numbers, dot, underscore, or hyphen (max 64)", + }); + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + issues.push({ path: [], message: "routing profile must be an object" }); + return issues; + } + const body = raw as Record; + + if (body.alias !== undefined) { + if (typeof body.alias !== "string") { + issues.push({ path: ["alias"], message: "alias must be a string" }); + } else { + const alias = body.alias.trim(); + if (alias) issues.push(...aliasIssues(id, alias, config, options)); + } + } + + if (!Array.isArray(body.candidates) || body.candidates.length === 0) { + issues.push({ path: ["candidates"], message: "candidates must be a non-empty array" }); + } else { + const seen = new Set(); + body.candidates.forEach((rawCandidate, index) => { + if (!rawCandidate || typeof rawCandidate !== "object" || Array.isArray(rawCandidate)) { + issues.push({ path: ["candidates", index], message: `candidates[${index}] must be an object` }); + return; + } + const candidate = rawCandidate as Record; + const provider = typeof candidate.provider === "string" ? candidate.provider.trim() : ""; + const model = typeof candidate.model === "string" ? candidate.model.trim() : ""; + if (!provider) { + issues.push({ path: ["candidates", index, "provider"], message: `candidates[${index}].provider is required` }); + } else if (!hasOwnProvider(config.providers, provider)) { + issues.push({ + path: ["candidates", index, "provider"], + message: `candidates[${index}].provider "${provider}" is not configured`, + }); + } else if (config.providers[provider]?.disabled === true) { + issues.push({ + path: ["candidates", index, "provider"], + message: `candidates[${index}].provider "${provider}" is disabled`, + }); + } + if (!model) { + issues.push({ path: ["candidates", index, "model"], message: `candidates[${index}].model is required` }); + } + if (provider && model) { + const key = `${provider}/${model}`; + if (seen.has(key)) { + issues.push({ path: ["candidates", index], message: `duplicate policy candidate "${key}"` }); + } else { + seen.add(key); + } + } + }); + } + + if (body.require !== undefined) { + if (!body.require || typeof body.require !== "object" || Array.isArray(body.require)) { + issues.push({ path: ["require"], message: "require must be an object" }); + } else { + const require = body.require as Record; + if (require.minContextWindow !== undefined + && (typeof require.minContextWindow !== "number" + || !Number.isInteger(require.minContextWindow) + || require.minContextWindow < 1)) { + issues.push({ path: ["require", "minContextWindow"], message: "minContextWindow must be a positive integer" }); + } + for (const key of ["tools", "imageInput", "structuredOutput", "localOnly", "remoteAllowed", "encryptedCodexTasks"] as const) { + if (require[key] !== undefined && typeof require[key] !== "boolean") { + issues.push({ path: ["require", key], message: `${key} must be a boolean` }); + } + } + if (require.reasoningEffort !== undefined && typeof require.reasoningEffort !== "string") { + issues.push({ path: ["require", "reasoningEffort"], message: "reasoningEffort must be a string" }); + } + if (require.serviceTier !== undefined && typeof require.serviceTier !== "string") { + issues.push({ path: ["require", "serviceTier"], message: "serviceTier must be a string" }); + } else if (require.serviceTier === "unknown") { + issues.push({ + path: ["require", "serviceTier"], + message: "serviceTier must not use the reserved \"unknown\" value (it encodes missing evidence)", + }); + } + } + } + + if (body.optimize !== undefined) { + if (!body.optimize || typeof body.optimize !== "object" || Array.isArray(body.optimize)) { + issues.push({ path: ["optimize"], message: "optimize must be an object" }); + } else { + const optimize = body.optimize as Record; + const effective: Record<"latency" | "health" | "cost" | "quota", number> = { ...DEFAULT_PROFILE_WEIGHTS }; + for (const key of ["latency", "health", "cost", "quota"] as const) { + if (optimize[key] !== undefined) { + if (typeof optimize[key] !== "number" + || !Number.isFinite(optimize[key]) + || optimize[key] < 0) { + issues.push({ path: ["optimize", key], message: `${key} must be a non-negative number` }); + } else { + effective[key] = optimize[key]; + } + } + } + if (Object.values(effective).every(weight => weight === 0)) { + issues.push({ path: ["optimize"], message: "at least one optimize weight must be positive" }); + } + } + } + + if (body.limits !== undefined) { + if (!body.limits || typeof body.limits !== "object" || Array.isArray(body.limits)) { + issues.push({ path: ["limits"], message: "limits must be an object" }); + } else { + const limits = body.limits as Record; + if (limits.maxEstimatedCostUsd !== undefined + && (typeof limits.maxEstimatedCostUsd !== "number" + || !Number.isFinite(limits.maxEstimatedCostUsd) + || limits.maxEstimatedCostUsd < 0)) { + issues.push({ path: ["limits", "maxEstimatedCostUsd"], message: "maxEstimatedCostUsd must be a non-negative number" }); + } + } + } + + if (body.unknownEvidence !== undefined) { + if (!body.unknownEvidence || typeof body.unknownEvidence !== "object" || Array.isArray(body.unknownEvidence)) { + issues.push({ path: ["unknownEvidence"], message: "unknownEvidence must be an object" }); + } else { + const unknownEvidence = body.unknownEvidence as Record; + for (const key of UNKNOWN_EVIDENCE_KEYS) { + if (unknownEvidence[key] !== undefined + && (unknownEvidence[key] !== "allow" + && unknownEvidence[key] !== "penalize" + && unknownEvidence[key] !== "exclude")) { + issues.push({ path: ["unknownEvidence", key], message: `${key} must be "allow", "penalize", or "exclude"` }); + } + } + } + } + + return issues; +} + +function normalizedRequirements(raw: OcxRoutingProfileConfig): NormalizedRoutingProfileRequirements { + const require = raw.require; + if (!require) return {}; + const out: NormalizedRoutingProfileRequirements = {}; + for (const key of REQUIRE_KEYS) { + const value = require[key]; + if (value !== undefined) { + (out as Record)[key] = value; + } + } + return out; +} + +function normalizedUnknownEvidence(raw: OcxRoutingProfileConfig): NormalizedRoutingProfile["unknownEvidence"] { + const configured = raw.unknownEvidence; + const out = { ...DEFAULT_UNKNOWN_EVIDENCE }; + if (configured) { + for (const key of UNKNOWN_EVIDENCE_KEYS) { + const value = configured[key]; + if (value === "allow" || value === "penalize" || value === "exclude") { + out[key] = value; + } + } + } + return out; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const record = value as Record; + const keys = Object.keys(record).sort(); + return `{${keys.map(key => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; +} + +function profileRevision(profile: Omit): string { + const digest = createHash("sha256").update(canonicalJson(profile)).digest("hex"); + return digest.slice(0, 16); +} + +export function normalizeRoutingProfile(id: string, raw: OcxRoutingProfileConfig): NormalizedRoutingProfile { + const alias = typeof raw.alias === "string" ? raw.alias.trim() : ""; + const weights = { ...DEFAULT_PROFILE_WEIGHTS, ...(raw.optimize ?? {}) }; + const weightSum = weights.latency + weights.health + weights.cost + weights.quota; + const safeSum = weightSum > 0 ? weightSum : 1; + const profile: Omit = { + id, + alias: alias || null, + candidates: raw.candidates.map(candidate => ({ + provider: candidate.provider.trim(), + model: candidate.model.trim(), + })), + require: normalizedRequirements(raw), + optimize: { + latency: weights.latency / safeSum, + health: weights.health / safeSum, + cost: weights.cost / safeSum, + quota: weights.quota / safeSum, + }, + limits: { + ...(raw.limits?.maxEstimatedCostUsd !== undefined + ? { maxEstimatedCostUsd: raw.limits.maxEstimatedCostUsd } + : {}), + }, + unknownEvidence: normalizedUnknownEvidence(raw), + }; + return { ...profile, revision: profileRevision(profile) }; +} + +export function getRoutingProfile( + config: { routingProfiles?: Record }, + id: string, +): NormalizedRoutingProfile | undefined { + const profiles = config.routingProfiles; + if (!profiles || !Object.hasOwn(profiles, id)) return undefined; + return normalizeRoutingProfile(id, profiles[id]!); +} + +export function listRoutingProfileIds(config: { routingProfiles?: Record }): string[] { + // Code-unit comparison: deterministic across ICU versions/platforms, which + // matters for the API/CLI list contract (ids may contain ".", "_", "-"). + return Object.keys(config.routingProfiles ?? {}).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); +} diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 2215e3655a..7a97d421cc 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -46,7 +46,8 @@ export interface RouteCapabilityEvidence { image?: Unknownable; structuredOutput?: Unknownable; reasoningEfforts?: string[]; - serviceTier?: Unknownable; + /** `"unknown"` is reserved: it encodes missing evidence, never a real tier. */ + serviceTier?: string | "unknown"; localOnly?: Unknownable; remoteAllowed?: Unknownable; encryptedCodexTasks?: Unknownable; @@ -178,6 +179,7 @@ export interface TraceCandidateInput { accountRef?: string; eligible: boolean; exclusions: RouteExclusionReason[]; + score?: RouteScoreEvidence; } export interface TraceBuildInput { @@ -215,6 +217,7 @@ function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; ex ? { detail: capString(exclusion.detail, budget) } : {}), })), + ...(input.score ? { score: input.score } : {}), }; } @@ -453,8 +456,12 @@ function parseCapability(raw: unknown, caps: ParseCaps): RouteCapabilityEvidence .slice(0, 8) .map(value => value.slice(0, MAX_TRACE_STRING)); } - const serviceTier = unknownable(raw.serviceTier); - if (serviceTier !== undefined) out.serviceTier = serviceTier; + if (raw.serviceTier === "unknown") { + out.serviceTier = "unknown"; + } else if (typeof raw.serviceTier === "string" && raw.serviceTier) { + if (raw.serviceTier.length > MAX_TRACE_STRING) caps.strings = true; + out.serviceTier = raw.serviceTier.slice(0, MAX_TRACE_STRING); + } const localOnly = unknownable(raw.localOnly); if (localOnly !== undefined) out.localOnly = localOnly; const remoteAllowed = unknownable(raw.remoteAllowed); diff --git a/src/server/management-api.ts b/src/server/management-api.ts index af8fd499db..61936bd2e4 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -61,6 +61,7 @@ import { handleConfigRoutes } from "./management/config-routes"; import { handleLogsUsageRoutes } from "./management/logs-usage-routes"; import { handleRequestHistoryRoutes } from "./management/request-history-routes"; import { handleRoutingAnalyticsRoutes } from "./management/routing-analytics-routes"; +import { handleRoutingProfileRoutes } from "./management/routing-profile-routes"; import { handleProviderRoutes } from "./management/provider-routes"; import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; @@ -142,6 +143,7 @@ export async function handleManagementAPI( ?? (await handleLogsUsageRoutes(ctx)) ?? (await handleRequestHistoryRoutes(ctx)) ?? (await handleRoutingAnalyticsRoutes(ctx)) + ?? (await handleRoutingProfileRoutes(ctx)) ?? (await handleProviderRoutes(ctx)) ?? (await handleModelRoutes(ctx)) ?? (await handleIntegrationRoutes(ctx)) diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts new file mode 100644 index 0000000000..78cc0a18c5 --- /dev/null +++ b/src/server/management/routing-profile-routes.ts @@ -0,0 +1,114 @@ +/** + * Routing-profile management API (RI-04). + * + * - `GET /api/routing-profiles` - normalized profiles with revisions + * - `POST /api/routing-profiles/dry-run` - deterministic dry-run evaluation + * (never dispatches an upstream request) + */ + +import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from "../../routing/profile"; +import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; +import { isPlainRecord } from "./shared"; +import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; +import { jsonResponse } from "../auth-cors"; +import type { ManagementContext } from "./context"; + +function profileDto(config: Parameters[0], id: string): Record | null { + const profile = getRoutingProfile(config, id); + if (!profile) return null; + return { + id, + model: policyPublicModelId(id, profile), + revision: profile.revision, + candidates: profile.candidates, + require: profile.require, + optimize: profile.optimize, + limits: profile.limits, + unknownEvidence: profile.unknownEvidence, + }; +} + +function parseEvidence(raw: unknown): { evidence: PolicyRequestEvidence; ok: boolean } { + // Absent evidence is empty evidence, mirroring the absent-candidates case. + if (raw === undefined) return { evidence: {}, ok: true }; + if (!isPlainRecord(raw)) return { evidence: {}, ok: false }; + const record = raw as Record; + const evidence: PolicyRequestEvidence = {}; + if (typeof record.contextWindow === "number" && Number.isFinite(record.contextWindow) && record.contextWindow >= 0) { + evidence.contextWindow = record.contextWindow; + } + for (const key of ["toolsRequired", "imageInputRequired", "structuredOutputRequired", "encryptedCodexTask"] as const) { + if (typeof record[key] === "boolean") evidence[key] = record[key]; + } + if (typeof record.reasoningEffort === "string") evidence.reasoningEffort = record.reasoningEffort; + if (typeof record.serviceTier === "string") evidence.serviceTier = record.serviceTier; + return { evidence, ok: true }; +} + +function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null { + if (!Array.isArray(raw)) return null; + const out: PolicyCandidateEvidence[] = []; + for (const item of raw) { + if (!isPlainRecord(item)) return null; + const provider = item.provider; + const model = item.model; + if (typeof provider !== "string" || typeof model !== "string") return null; + out.push({ + provider, + model, + ...(typeof item.accountRef === "string" ? { accountRef: item.accountRef } : {}), + // Dry-run evidence is caller-supplied and echoed back in the result as + // given; the trace's candidate rows carry only score/exclusions, which + // the trace builder bounds. Structural casts keep the API permissive. + ...(isPlainRecord(item.capability) ? { capability: item.capability as unknown as PolicyCandidateEvidence["capability"] } : {}), + ...(isPlainRecord(item.health) ? { health: item.health as unknown as PolicyCandidateEvidence["health"] } : {}), + ...(isPlainRecord(item.quota) ? { quota: item.quota as unknown as PolicyCandidateEvidence["quota"] } : {}), + ...(isPlainRecord(item.cost) ? { cost: item.cost as unknown as PolicyCandidateEvidence["cost"] } : {}), + }); + } + return out; +} + +export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promise { + const { req, url, config } = ctx; + + if (url.pathname === "/api/routing-profiles" && req.method === "GET") { + const profiles = listRoutingProfileIds(config).map(id => profileDto(config, id)).filter( + (profile): profile is Record => profile !== null, + ); + return jsonResponse({ profiles }, 200, req, config); + } + + if (url.pathname === "/api/routing-profiles/dry-run" && req.method === "POST") { + let rawBody: unknown; + try { rawBody = await readManagementJsonBody(req); } catch (error) { + rethrowManagementBodyTooLarge(error); + return jsonResponse({ error: "invalid JSON body" }, 400, req, config); + } + if (!isPlainRecord(rawBody)) { + return jsonResponse({ error: "request body must be an object" }, 400, req, config); + } + const body = rawBody as Record; + const profile = typeof body.profile === "string" ? body.profile.trim() : ""; + if (!profile) { + return jsonResponse({ error: { code: "missing_profile", message: "profile is required" } }, 400, req, config); + } + if (!getRoutingProfile(config, profile)) { + return jsonResponse({ error: { code: "unknown_profile", message: `unknown routing profile: ${profile}` } }, 404, req, config); + } + const { evidence, ok } = parseEvidence(body.evidence); + if (!ok) { + return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config); + } + const candidateEvidence = body.candidates === undefined + ? [] + : parseCandidateEvidence(body.candidates); + if (candidateEvidence === null) { + return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config); + } + const result = evaluatePolicyProfile(config, profile, evidence, candidateEvidence); + return jsonResponse(result, 200, req, config); + } + + return null; +} diff --git a/src/types.ts b/src/types.ts index ae84aa6740..1774461579 100644 --- a/src/types.ts +++ b/src/types.ts @@ -773,6 +773,13 @@ export interface OcxConfig { }; /** Virtual `combo/` models spanning concrete provider/model targets (issue #133). */ combos?: Record; + /** + * Routing policy profiles (Router Intelligence, RI-04+): explicitly requested + * `policy/` (or configured alias) models select among an explicit + * candidate allowlist using hard capability requirements and deterministic + * scoring. Existing model ids are never routed through profiles implicitly. + */ + routingProfiles?: Record; /** Background proactive token refresh ("Token Guardian"). Off by default; see OcxTokenGuardianConfig. */ tokenGuardian?: OcxTokenGuardianConfig; /** Additional exact origins allowed for CORS (e.g. HTTPS or chrome-extension://). Loopback origins are always allowed. */ @@ -807,6 +814,63 @@ export interface OcxComboConfig { alias?: string; } +export type OcxRoutingUnknownEvidenceMode = "allow" | "penalize" | "exclude"; + +export interface OcxRoutingProfileCandidate { + provider: string; + model: string; +} + +export interface OcxRoutingProfileRequirements { + /** Minimum model context window in tokens. */ + minContextWindow?: number; + tools?: boolean; + imageInput?: boolean; + structuredOutput?: boolean; + reasoningEffort?: string; + serviceTier?: string; + localOnly?: boolean; + remoteAllowed?: boolean; + /** Special encrypted Codex task readability (ChatGPT forward pool). */ + encryptedCodexTasks?: boolean; +} + +export interface OcxRoutingProfileOptimize { + latency?: number; + health?: number; + cost?: number; + quota?: number; +} + +export interface OcxRoutingProfileLimits { + /** Hard per-request estimated-cost ceiling in USD. */ + maxEstimatedCostUsd?: number; +} + +export interface OcxRoutingProfileUnknownEvidence { + capability?: OcxRoutingUnknownEvidenceMode; + health?: OcxRoutingUnknownEvidenceMode; + quota?: OcxRoutingUnknownEvidenceMode; + cost?: OcxRoutingUnknownEvidenceMode; +} + +export interface OcxRoutingProfileConfig { + /** + * Explicit candidate allowlist (`provider/model` refs). No implicit + * expansion in v1. + */ + candidates: OcxRoutingProfileCandidate[]; + /** Optional public model name replacing the default `policy/` slug. */ + alias?: string; + /** Hard requirements evaluated before scoring. */ + require?: OcxRoutingProfileRequirements; + /** Optimization weights; normalized deterministically. */ + optimize?: OcxRoutingProfileOptimize; + limits?: OcxRoutingProfileLimits; + /** How unknown evidence is handled per dimension. */ + unknownEvidence?: OcxRoutingProfileUnknownEvidence; +} + /** * Per-provider proactive-refresh policy. The guardian only ever touches a provider whose EFFECTIVE * policy is "proactive"; "lazy-only" keeps today's on-demand refresh, "disabled" forbids the diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts new file mode 100644 index 0000000000..4865e56876 --- /dev/null +++ b/tests/routing-profile.test.ts @@ -0,0 +1,364 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { validateConfigCandidate } from "../src/config"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + getRoutingProfile, + listRoutingProfileIds, + normalizeRoutingProfile, + parsePolicyModelId, + policyPublicModelId, + resolvePolicyProfileId, + routingProfileIssues, +} from "../src/routing/profile"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-profile-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + }, + combos: { free: { strategy: "failover", targets: [{ provider: "a", model: "m1" }] } }, + codexAccountNamespaces: { work: "acct-1" }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + optimize: { latency: 0.55, health: 0.25, cost: 0.10, quota: 0.10 }, + limits: { maxEstimatedCostUsd: 0.5 }, + unknownEvidence: { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + ...overrides, + }; +} + +describe("routing profiles (RI-04)", () => { + test("normalizes a valid profile with deterministic weights and revision", () => { + const profile = getRoutingProfile(baseConfig(), "fast")!; + expect(profile.id).toBe("fast"); + expect(profile.alias).toBe("ocx/fast"); + expect(profile.candidates).toEqual([ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ]); + expect(profile.require).toMatchObject({ tools: true, minContextWindow: 128000 }); + expect(profile.optimize.latency + profile.optimize.health + profile.optimize.cost + profile.optimize.quota).toBeCloseTo(1); + expect(profile.limits.maxEstimatedCostUsd).toBe(0.5); + expect(profile.revision).toMatch(/^[0-9a-f]{16}$/); + }); + + test("revision digest is stable and changes with the profile", () => { + const config = baseConfig(); + const first = getRoutingProfile(config, "fast")!.revision; + const second = getRoutingProfile(config, "fast")!.revision; + expect(first).toBe(second); + const changed = baseConfig({ + routingProfiles: { + fast: { + ...config.routingProfiles!.fast!, + candidates: [{ provider: "a", model: "m1" }], + }, + }, + }); + expect(getRoutingProfile(changed, "fast")!.revision).not.toBe(first); + }); + + test("weights default and normalize deterministically", () => { + const config = baseConfig({ + routingProfiles: { only: { candidates: [{ provider: "a", model: "m1" }] } }, + }); + const profile = getRoutingProfile(config, "only")!; + expect(profile.optimize).toEqual({ latency: 0.55, health: 0.25, cost: 0.1, quota: 0.1 }); + const weighted = baseConfig({ + routingProfiles: { w: { candidates: [{ provider: "a", model: "m1" }], optimize: { latency: 1, cost: 3 } } }, + }); + const normalized = getRoutingProfile(weighted, "w")!; + // Unspecified weights keep their defaults: latency 1, health 0.25, + // cost 3, quota 0.1 => sum 4.35, normalized deterministically. + expect(normalized.optimize.latency).toBeCloseTo(1 / 4.35); + expect(normalized.optimize.cost).toBeCloseTo(3 / 4.35); + const sum = normalized.optimize.latency + normalized.optimize.health + + normalized.optimize.cost + normalized.optimize.quota; + expect(sum).toBeCloseTo(1); + + const allZero = routingProfileIssues("z", { + candidates: [{ provider: "a", model: "m1" }], + optimize: { latency: 0, health: 0, cost: 0, quota: 0 }, + }, config); + expect(allZero.some(issue => issue.path.join(".") === "optimize")).toBe(true); + // A partial zero is fine: unspecified weights keep their positive defaults. + const partialZero = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + optimize: { latency: 0 }, + }, config); + expect(partialZero.some(issue => issue.path.join(".") === "optimize")).toBe(false); + }); + + test("alias collision validation covers providers, combos, account namespaces, native families", () => { + const config = baseConfig(); + const providerCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "a", + }, config); + expect(providerCollision.some(issue => issue.message.includes("provider name"))).toBe(true); + + const comboCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "combo/free", + }, config); + expect(comboCollision.some(issue => issue.message.includes("reserved"))).toBe(true); + + const comboAliasCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "faster", + }, { + ...config, + combos: { ...config.combos, free: { strategy: "failover", targets: [{ provider: "a", model: "m1" }], alias: "faster" } }, + }); + expect(comboAliasCollision.some(issue => issue.message.includes("combo selector"))).toBe(true); + + const nativeCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "gpt-5.6", + }, config); + expect(nativeCollision.some(issue => issue.message.includes("native family"))).toBe(true); + + const providerNamespaceCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "a/m1", + }, config); + expect(providerNamespaceCollision.some(issue => issue.message.includes("provider routing namespace"))).toBe(true); + + const siblingCollision = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + alias: "ocx/fast", + }, config); + expect(siblingCollision.some(issue => issue.message.includes("already used"))).toBe(true); + }); + + test("candidate validation rejects unconfigured/disabled providers and duplicates", () => { + const config = baseConfig(); + const unconfigured = routingProfileIssues("p", { + candidates: [{ provider: "ghost", model: "m1" }], + }, config); + expect(unconfigured.some(issue => issue.message.includes("not configured"))).toBe(true); + + const disabled = baseConfig({ + providers: { ...baseConfig().providers, c: { adapter: "openai-chat", baseUrl: "https://c.example/v1", apiKey: "kc", models: ["m3"], disabled: true } }, + routingProfiles: { p: { candidates: [{ provider: "c", model: "m3" }] } }, + }); + expect(routingProfileIssues("p", disabled.routingProfiles!.p, disabled).some(issue => issue.message.includes("disabled"))).toBe(true); + + const duplicates = routingProfileIssues("p", { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "a", model: "m1" }, + ], + }, config); + expect(duplicates.some(issue => issue.message.includes("duplicate"))).toBe(true); + }); + + test("require rejects the reserved unknown service tier", () => { + const config = baseConfig(); + const reservedTier = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m1" }], + require: { serviceTier: "unknown" }, + }, config); + expect(reservedTier.some(issue => issue.path.join(".") === "require.serviceTier")).toBe(true); + }); + + test("config load accepts valid profiles and rejects broken ones", () => { + const valid = validateConfigCandidate(baseConfig()); + expect(valid.ok).toBe(true); + + const broken = validateConfigCandidate(baseConfig({ + routingProfiles: { bad: { candidates: [{ provider: "ghost", model: "m1" }] } }, + })); + expect(broken.ok).toBe(false); + if (!broken.ok) expect(broken.error).toContain("routingProfiles"); + }); + + test("policy id/alias resolution follows canonical-id-first", () => { + const config = baseConfig(); + expect(resolvePolicyProfileId(config, "policy/fast")).toBe("fast"); + expect(resolvePolicyProfileId(config, "ocx/fast")).toBe("fast"); + expect(resolvePolicyProfileId(config, "policy/missing")).toBe("missing"); + expect(resolvePolicyProfileId(config, "unknown")).toBeNull(); + expect(parsePolicyModelId("policy/fast")).toBe("fast"); + expect(parsePolicyModelId("a/m1")).toBeNull(); + expect(policyPublicModelId("fast", getRoutingProfile(config, "fast")!)).toBe("ocx/fast"); + }); + + test("dry-run evaluator: hard requirements gate eligibility", () => { + const config = baseConfig(); + const result = evaluatePolicyProfile(config, "fast", { contextWindow: 200000, toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + { provider: "b", model: "m2", capability: { contextWindow: 64000, tools: true } }, + ]); + expect(result.candidates.length).toBe(2); + expect(result.candidates[0]).toMatchObject({ eligible: true }); + expect(result.candidates[1]).toMatchObject({ eligible: false }); + expect(result.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); + expect(result.selectedIndex).toBe(0); + expect(result.trace.routeKind).toBe("policy"); + expect(result.trace.profile).toEqual({ id: "fast", revision: result.profileRevision }); + expect(result.trace.selected.provider).toBe("a"); + expect(result.trace.selected.model).toBe("m1"); + expect(result.trace.selected.reason).toBe("policy-selected"); + }); + + test("dry-run evaluator: unknown capability follows the profile's unknownEvidence", () => { + const config = baseConfig(); + const unknownEvidence = { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }; + const strict = baseConfig({ + routingProfiles: { strict: { candidates: [{ provider: "a", model: "m1" }], require: { tools: true }, unknownEvidence } }, + providers: baseConfig().providers, + }); + const excluded = evaluatePolicyProfile(strict, "strict", { toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-capability")).toBe(true); + expect(excluded.selectedIndex).toBeNull(); + + const permissive = baseConfig({ + routingProfiles: { permissive: { candidates: [{ provider: "a", model: "m1" }], require: { tools: true }, unknownEvidence: { ...unknownEvidence, capability: "allow" } } }, + }); + const allowed = evaluatePolicyProfile(permissive, "permissive", { toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(allowed.candidates[0]!.eligible).toBe(true); + expect(allowed.selectedIndex).toBe(0); + }); + + test("dry-run evaluator: request evidence joins profile requirements", () => { + const config = baseConfig(); + // Profile `fast` requires tools + 128k context; the request also needs 256k. + const result = evaluatePolicyProfile(config, "fast", { contextWindow: 256000, toolsRequired: true }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + { provider: "b", model: "m2", capability: { contextWindow: 300000, tools: true } }, + ]); + expect(result.candidates[0]!.eligible).toBe(false); + expect(result.candidates[0]!.exclusions.some(exclusion => exclusion.detail === "request-context-window")).toBe(true); + expect(result.candidates[1]!.eligible).toBe(true); + expect(result.selectedIndex).toBe(1); + expect(result.trace.selected.model).toBe("m2"); + }); + + test("dry-run evaluator: absent request flags add no requirements", () => { + const config = baseConfig(); + const result = evaluatePolicyProfile(config, "fast", { toolsRequired: false }, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + ]); + expect(result.candidates[0]!.requirements.some(requirement => requirement.id === "request-tools")).toBe(false); + // Profile `fast` still requires tools; the candidate satisfies it. + expect(result.candidates[0]!.eligible).toBe(true); + expect(result.selectedIndex).toBe(0); + }); + + test("dry-run evaluator: cost limit excludes over-limit candidates", () => { + const config = baseConfig(); + const result = evaluatePolicyProfile(config, "fast", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true }, cost: { estimatedUsd: 1.2 } }, + { provider: "b", model: "m2", capability: { contextWindow: 200000, tools: true }, cost: { estimatedUsd: 0.2 } }, + ]); + expect(result.candidates[0]!.eligible).toBe(false); + expect(result.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cost-limit")).toBe(true); + expect(result.candidates[1]!.eligible).toBe(true); + expect(result.selectedIndex).toBe(1); + }); + + test("dry-run evaluator: deterministic priority picks the earlier candidate", () => { + const config = baseConfig({ + routingProfiles: { tie: { candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], require: { minContextWindow: 1000 } } }, + }); + const result = evaluatePolicyProfile(config, "tie", { contextWindow: 2000 }, [ + { provider: "a", model: "m1", capability: { contextWindow: 5000 } }, + { provider: "b", model: "m2", capability: { contextWindow: 5000 } }, + ]); + expect(result.selectedIndex).toBe(0); + expect(result.trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + }); + + test("API lists profiles and dry-runs deterministically", async () => { + const config = baseConfig(); + const listReq = new ManagementRequest("http://localhost/api/routing-profiles", { method: "GET" }); + const listResponse = await handleManagementAPI(listReq, new URL(listReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(listResponse).not.toBeNull(); + expect(listResponse!.status).toBe(200); + const listBody = await listResponse!.json() as { profiles?: Array<{ id?: string; revision?: string }> }; + expect(listBody.profiles?.length).toBe(1); + expect(listBody.profiles![0]).toMatchObject({ + id: "fast", + model: "ocx/fast", + revision: getRoutingProfile(config, "fast")!.revision, + }); + + const dryReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "fast", + evidence: { contextWindow: 200000, toolsRequired: true }, + candidates: [ + { provider: "a", model: "m1", capability: { contextWindow: 200000, tools: true } }, + { provider: "b", model: "m2", capability: { contextWindow: 64000, tools: true } }, + ], + }), + }); + const dryResponse = await handleManagementAPI(dryReq, new URL(dryReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(dryResponse!.status).toBe(200); + const dryBody = await dryResponse!.json() as { selectedIndex?: number | null; trace?: { selected?: { provider?: string } } }; + expect(dryBody.selectedIndex).toBe(0); + expect(dryBody.trace?.selected?.provider).toBe("a"); + }); + + test("API dry-run rejects unknown profiles and invalid evidence", async () => { + const config = baseConfig(); + const unknownReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "nope", evidence: {} }), + }); + const unknownResponse = await handleManagementAPI(unknownReq, new URL(unknownReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(unknownResponse!.status).toBe(404); + + const badReq = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", evidence: "junk" }), + }); + const badResponse = await handleManagementAPI(badReq, new URL(badReq.url), config, { refreshCodexCatalog: async () => {} }); + expect(badResponse!.status).toBe(400); + }); +});