From b562fdb927447e82389f45b3bec91a12d1eb628f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:04:17 +0200 Subject: [PATCH 1/8] feat(routing): add quota-aware policy scoring (RI-07) --- .../001_pr_stack_status.md | 4 +- .../docs/reference/configuration/routing.md | 6 +- src/router.ts | 9 + src/routing/evaluator.ts | 41 ++++- src/routing/profile.ts | 12 +- src/routing/quota.ts | 88 ++++++++++ src/routing/trace.ts | 3 + src/types.ts | 2 + tests/quota-scoring.test.ts | 155 ++++++++++++++++++ 9 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 src/routing/quota.ts create mode 100644 tests/quota-scoring.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index c0e6ebf87..07f4015f9 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -45,8 +45,8 @@ other; closing one is a maintainer decision and neither is stale. | RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | `a594938c5` | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | MERGED | | RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | MERGED | | RI-05 | `feat/ri-05-capability-aware-routing` | `dev` (post-#1011 merge) | `088194a3a` | #1012 | https://github.com/lidge-jun/opencodex/pull/1012 | MERGED | -| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | in progress | -| RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | +| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | MERGED | +| RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `0c5100271` (pre-restack) | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | in progress | | RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | | RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued | | RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued | diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 780f815ab..bc71a1852 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -108,9 +108,9 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | -`require` supports: `minContextWindow` (positive integer), and the booleans `tools`, `imageInput`, -`structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`; plus `reasoningEffort` and -`serviceTier` strings. +`require` supports: `minContextWindow` (positive integer), `minQuotaHeadroom` (0..1 fraction), +and the booleans `tools`, `imageInput`, `structuredOutput`, `localOnly`, `remoteAllowed`, +`encryptedCodexTasks`; plus `reasoningEffort` and `serviceTier` strings. For `unknownEvidence.capability`, `penalize` currently behaves like `allow`: scoring has only a configured-priority component until a capability score dimension ships (planned with RI-06+), so diff --git a/src/router.ts b/src/router.ts index 61a25b845..a8910b8bf 100644 --- a/src/router.ts +++ b/src/router.ts @@ -22,6 +22,7 @@ import { import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec"; import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; +import { getEffectiveActiveCodexAccountId } from "./codex/routing"; import { buildRouteDecisionTrace, type RouteDecisionKind, @@ -32,6 +33,7 @@ import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; import { candidateCapabilityEvidence } from "./routing/capability"; import { policyCandidateHealthEvidence } from "./routing/health"; +import { quotaEvidenceForCandidate } from "./routing/quota"; export class NoEligiblePolicyCandidateError extends Error { /** Evaluation trace (with per-candidate exclusions) when nothing qualified. */ @@ -506,6 +508,13 @@ function routeModelInternal( model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), health: policyCandidateHealthEvidence(config, candidate, now), + quota: quotaEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ? getEffectiveActiveCodexAccountId(config) + : undefined, + }), })); const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence, now); if (evaluation.selectedIndex === null) { diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 363f97d06..6b97e0a70 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -22,11 +22,14 @@ import { } from "./trace"; import { getRoutingProfile, policyModelId, type NormalizedRoutingProfile } from "./profile"; import { healthScore } from "./health"; +import { quotaScore } from "./quota"; /** Unknown health under "penalize": a low-but-not-zero deterministic floor. */ export const HEALTH_UNKNOWN_PENALTY_SCORE = 0.3; /** Unknown health under "allow": neutral midpoint of the [0,1] health scale. */ export const HEALTH_UNKNOWN_NEUTRAL_SCORE = 0.5; +/** Unknown quota under "penalize": deterministic low floor. */ +export const QUOTA_UNKNOWN_PENALTY_SCORE = 0.3; export interface PolicyRequestEvidence { /** Required context window for this request (tokens). */ @@ -93,6 +96,7 @@ function booleanRequirement( function requirementFor( require: NormalizedRoutingProfile["require"], capability: RouteCapabilityEvidence | undefined, + quota: RouteQuotaEvidence | undefined, ): RouteRequirementEvidence[] { const requirements: RouteRequirementEvidence[] = []; if (require.minContextWindow !== undefined) { @@ -108,6 +112,17 @@ function requirementFor( requirements.push({ id: "min-context-window", expected: require.minContextWindow, outcome: "unknown" }); } } + // minQuotaHeadroom gates only KNOWN headroom. Unknown quota is governed by + // the profile's `unknownEvidence.quota` policy (exclude / penalize / allow) + // via the quota score path - never by the capability unknown policy. + if (require.minQuotaHeadroom !== undefined && typeof quota?.headroom === "number") { + requirements.push({ + id: "min-quota-headroom", + expected: require.minQuotaHeadroom, + actual: quota.headroom, + outcome: quota.headroom >= require.minQuotaHeadroom ? "satisfied" : "unsatisfied", + }); + } const tools = booleanRequirement("tools", require.tools, capability?.tools); if (tools) requirements.push(tools); const image = booleanRequirement("image-input", require.imageInput, capability?.image); @@ -258,7 +273,7 @@ export function evaluatePolicyProfile( candidate => candidate.provider === declared.provider && candidate.model === declared.model, ) ?? { provider: declared.provider, model: declared.model }; const requirements = [ - ...requirementFor(profile.require, evidence.capability), + ...requirementFor(profile.require, evidence.capability, evidence.quota), ...requestRequirementFor(requestEvidence, evidence.capability), ]; const exclusions: RouteExclusionReason[] = []; @@ -307,14 +322,34 @@ export function evaluatePolicyProfile( healthValue = HEALTH_UNKNOWN_NEUTRAL_SCORE; } + // Quota scoring (RI-07): unknown quota follows the profile policy; + // exhausted or low-headroom evidence lowers the score. + const quota = evidence.quota; + let quotaValue = quota ? quotaScore(quota) : null; + if (quotaValue === null) { + if (profile.unknownEvidence.quota === "exclude") { + exclusions.push({ code: "unknown-quota" }); + eligible = false; + } else if (profile.unknownEvidence.quota === "penalize") { + quotaValue = QUOTA_UNKNOWN_PENALTY_SCORE; + } + } + const priorityScore = configuredPriorityScore(index, profile.candidates.length); const healthWeight = profile.optimize.health; + const quotaWeight = profile.optimize.quota; + const implementedWeight = healthWeight + quotaWeight; + const priorityWeight = Math.max(0, 1 - implementedWeight); const components: RouteScoreEvidence["components"] = { configuredPriority: priorityScore }; - let total = priorityScore; + let total = priorityWeight * priorityScore; if (healthWeight > 0 && healthValue !== null) { - total = priorityScore * (1 - healthWeight) + healthValue * healthWeight; + total += healthWeight * healthValue; components.health = healthValue; } + if (quotaWeight > 0 && quotaValue !== null) { + total += quotaWeight * quotaValue; + components.quota = quotaValue; + } const score: RouteScoreEvidence = { total, components }; const evaluated: PolicyEvaluationCandidate = { provider: evidence.provider, diff --git a/src/routing/profile.ts b/src/routing/profile.ts index bc5ac85c3..4f49c07ae 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -41,6 +41,8 @@ export interface RoutingProfileValidationIssue { export interface NormalizedRoutingProfileRequirements { minContextWindow?: number; + /** Minimum remaining quota headroom fraction (0..1). */ + minQuotaHeadroom?: number; tools?: boolean; imageInput?: boolean; structuredOutput?: boolean; @@ -64,6 +66,7 @@ export interface NormalizedRoutingProfile { const REQUIRE_KEYS = [ "minContextWindow", + "minQuotaHeadroom", "tools", "imageInput", "structuredOutput", @@ -247,12 +250,19 @@ export function routingProfileIssues( issues.push({ path: ["require"], message: "require must be an object" }); } else { const require = body.require as Record; - if (require.minContextWindow !== undefined + if (require.minContextWindow !== undefined && (typeof require.minContextWindow !== "number" || !Number.isInteger(require.minContextWindow) || require.minContextWindow < 1)) { issues.push({ path: ["require", "minContextWindow"], message: "minContextWindow must be a positive integer" }); } + if (require.minQuotaHeadroom !== undefined + && (typeof require.minQuotaHeadroom !== "number" + || !Number.isFinite(require.minQuotaHeadroom) + || require.minQuotaHeadroom < 0 + || require.minQuotaHeadroom > 1)) { + issues.push({ path: ["require", "minQuotaHeadroom"], message: "minQuotaHeadroom must be a number from 0 to 1" }); + } for (const key of ["tools", "imageInput", "structuredOutput", "localOnly", "remoteAllowed", "encryptedCodexTasks"] as const) { if (require[key] !== undefined && typeof require[key] !== "boolean") { issues.push({ path: ["require", key], message: `${key} must be a boolean` }); diff --git a/src/routing/quota.ts b/src/routing/quota.ts new file mode 100644 index 000000000..d729d53f7 --- /dev/null +++ b/src/routing/quota.ts @@ -0,0 +1,88 @@ +/** + * Quota-aware policy scoring (RI-07). + * + * Evidence reuses the existing privacy-safe quota sources: the Codex pool + * account quota cache (usage percents only - no emails, tokens, or raw + * responses) and the provider per-account quota cache (Anthropic). Unknown + * quota stays unknown; no raw quota response ever reaches a trace. + * + * Boundary: policy profiles select provider/model TARGETS. Exact account + * selectors and Codex pool strategies remain authoritative inside their + * existing scope; account-level quota evidence is consumed when a candidate + * carries an account reference (dry-run/evaluate), never invented. + */ + +import { getAccountQuota, isCodexQuotaExhausted } from "../codex/quota"; +import { getCachedProviderAccountQuota } from "../providers/quota"; +import type { RouteQuotaEvidence } from "./trace"; + +export interface QuotaEvidenceInput { + provider: string; + model: string; + /** Opaque account reference for per-account quota sources. */ + accountRef?: string; + /** Codex pool account id (provider "openai"). */ + codexAccountId?: string; +} + +/** + * Assemble quota evidence from canonical local caches only (no network). + * Unknown dimensions stay unknown - never zero. + */ +export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuotaEvidence { + if (input.provider === "openai" && input.codexAccountId) { + const quota = getAccountQuota(input.codexAccountId); + if (quota) { + const percents = [quota.weeklyPercent, quota.monthlyPercent] + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; + const resets = [quota.weeklyResetAt, quota.monthlyResetAt] + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)) + .filter(value => value > Date.now()); + return { + known: true, + ...(maxPercent !== undefined + ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } + : {}), + exhausted: isCodexQuotaExhausted(quota), + ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), + source: "codex-pool", + }; + } + } + + if (input.provider === "anthropic" && input.accountRef) { + const quota = getCachedProviderAccountQuota("anthropic", input.accountRef); + if (quota) { + const percents = [quota.fiveHourPercent, quota.weeklyPercent, quota.monthlyPercent] + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; + const resets = [quota.fiveHourResetAt, quota.weeklyResetAt, quota.monthlyResetAt] + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)) + .filter(value => value > Date.now()); + return { + known: true, + ...(maxPercent !== undefined + ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } + : {}), + exhausted: maxPercent !== undefined && maxPercent >= 100, + ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), + source: "provider-report", + }; + } + } + + return { known: false }; +} + +/** + * Deterministic quota score in [0,1]: larger available headroom scores + * higher; exhausted evidence scores 0. Unknown evidence returns null so the + * caller can apply the profile's unknownEvidence policy. + */ +export function quotaScore(evidence: RouteQuotaEvidence | undefined): number | null { + if (!evidence || !evidence.known) return null; + if (evidence.exhausted === true) return 0; + if (typeof evidence.headroom === "number") return Math.max(0, Math.min(1, evidence.headroom)); + return null; +} diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 658c66b1a..0b67f2aa6 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -66,6 +66,8 @@ export interface RouteHealthEvidence { export interface RouteQuotaEvidence { known: boolean; + /** Remaining headroom as a fraction 0..1 (larger is better). */ + headroom?: number; headroomTokens?: number; exhausted?: boolean; resetAtMs?: number; @@ -507,6 +509,7 @@ function parseQuota(raw: unknown, caps: ParseCaps): RouteQuotaEvidence | undefin if (!isPlainRecord(raw)) return undefined; if (typeof raw.known !== "boolean") return undefined; const out: RouteQuotaEvidence = { known: raw.known }; + if (finiteNumber(raw.headroom)) out.headroom = Math.max(0, Math.min(1, raw.headroom)); if (finiteNumber(raw.headroomTokens)) out.headroomTokens = raw.headroomTokens; if (typeof raw.exhausted === "boolean") out.exhausted = raw.exhausted; if (finiteNumber(raw.resetAtMs)) out.resetAtMs = raw.resetAtMs; diff --git a/src/types.ts b/src/types.ts index 67b3f8bec..ad58b2d2f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -824,6 +824,8 @@ export interface OcxRoutingProfileCandidate { export interface OcxRoutingProfileRequirements { /** Minimum model context window in tokens. */ minContextWindow?: number; + /** Minimum remaining quota headroom fraction (0..1). */ + minQuotaHeadroom?: number; tools?: boolean; imageInput?: boolean; structuredOutput?: boolean; diff --git a/tests/quota-scoring.test.ts b/tests/quota-scoring.test.ts new file mode 100644 index 000000000..54eeb8329 --- /dev/null +++ b/tests/quota-scoring.test.ts @@ -0,0 +1,155 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { updateAccountQuota } from "../src/codex/quota"; +import { setCachedProviderAccountQuotaForTests, clearAccountQuotaCache } from "../src/providers/quota"; +import { quotaEvidenceForCandidate, quotaScore } from "../src/routing/quota"; +import { evaluatePolicyProfile, QUOTA_UNKNOWN_PENALTY_SCORE } from "../src/routing/evaluator"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-quota-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + clearAccountQuotaCache(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + routingProfiles: { + quota: { + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + optimize: { quota: 0.8 }, + }, + }, + ...overrides, + }; +} + +describe("quota-aware scoring (RI-07)", () => { + test("codex pool quota evidence derives headroom and exhausted state", async () => { + updateAccountQuota("acct-1", 30, 1_800_000_000_000, 20, 1_900_000_000_000); + const evidence = quotaEvidenceForCandidate({ provider: "openai", model: "gpt-5.6", codexAccountId: "acct-1" }); + expect(evidence.known).toBe(true); + expect(evidence.headroom).toBeCloseTo(0.7, 2); + expect(evidence.exhausted).toBe(false); + expect(evidence.source).toBe("codex-pool"); + expect(quotaScore(evidence)).toBeCloseTo(0.7, 2); + + updateAccountQuota("acct-2", 100, undefined, undefined); + const exhausted = quotaEvidenceForCandidate({ provider: "openai", model: "gpt-5.6", codexAccountId: "acct-2" }); + expect(exhausted.exhausted).toBe(true); + expect(quotaScore(exhausted)).toBe(0); + }); + + test("anthropic account quota evidence uses the provider cache", async () => { + setCachedProviderAccountQuotaForTests("anthropic", "acct-x", { + fiveHourPercent: 40, + fiveHourResetAt: 1_800_000_000_000, + updatedAt: Date.now(), + }); + const evidence = quotaEvidenceForCandidate({ provider: "anthropic", model: "claude-sonnet-5", accountRef: "acct-x" }); + expect(evidence.known).toBe(true); + expect(evidence.headroom).toBeCloseTo(0.6, 2); + expect(evidence.source).toBe("provider-report"); + expect(evidence.resetAtMs).toBe(1_800_000_000_000); + }); + + test("unknown quota stays unknown and never zero", () => { + const evidence = quotaEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.known).toBe(false); + expect(quotaScore(evidence)).toBeNull(); + }); + + test("unknown quota follows the profile policy (exclude / penalize / allow)", () => { + const strict = config({ + routingProfiles: { + q: { + candidates: [{ provider: "a", model: "m1" }], + unknownEvidence: { capability: "allow", health: "penalize", quota: "exclude", cost: "penalize" }, + }, + }, + }); + const excluded = evaluatePolicyProfile(strict, "q", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-quota")).toBe(true); + + const penalizing = config({ + routingProfiles: { + q: { + candidates: [{ provider: "a", model: "m1" }], + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const penalized = evaluatePolicyProfile(penalizing, "q", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(penalized.candidates[0]!.eligible).toBe(true); + expect(penalized.candidates[0]!.score!.components.quota).toBe(QUOTA_UNKNOWN_PENALTY_SCORE); + }); + + test("larger headroom is preferred when quota scoring is weighted", () => { + const result = evaluatePolicyProfile(config(), "quota", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, quota: { known: true, headroom: 0.2 } }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 }, quota: { known: true, headroom: 0.9 } }, + ]); + expect(result.selectedIndex).toBe(1); + expect(result.candidates[1]!.score!.components.quota).toBeCloseTo(0.9, 2); + expect(result.candidates[0]!.score!.components.quota).toBeCloseTo(0.2, 2); + }); + + test("minQuotaHeadroom hard requirement gates eligibility", () => { + const gated = config({ + routingProfiles: { + g: { + candidates: [{ provider: "a", model: "m1" }], + require: { minQuotaHeadroom: 0.5 }, + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const low = evaluatePolicyProfile(gated, "g", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, quota: { known: true, headroom: 0.2 } }, + ]); + expect(low.candidates[0]!.eligible).toBe(false); + expect(low.candidates[0]!.exclusions.some(exclusion => exclusion.code === "capability-unsatisfied")).toBe(true); + + const enough = evaluatePolicyProfile(gated, "g", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 }, quota: { known: true, headroom: 0.7 } }, + ]); + expect(enough.candidates[0]!.eligible).toBe(true); + }); + + test("exact account selectors and pool strategies remain authoritative", () => { + // Policy execution never invents account selection: candidates without + // account refs get unknown quota evidence and the profile policy decides. + const route = evaluatePolicyProfile(config(), "quota", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + { provider: "b", model: "m2", capability: { contextWindow: 200000 } }, + ]); + expect(route.candidates.every(candidate => candidate.quota === undefined || candidate.quota.known === false)).toBe(true); + }); +}); From 288dd8a90a06b1cb51535b2605af07d64c123e92 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 05:55:31 +0200 Subject: [PATCH 2/8] fix(routing): address full-review findings and approved simplify (RI-07) --- .../001_pr_stack_status.md | 51 ++++++++++++++++++ src/routing/profile.ts | 2 +- .../management/routing-profile-routes.ts | 5 ++ tests/quota-scoring.test.ts | 52 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 07f4015f9..4de69ce8f 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -214,3 +214,54 @@ other; closing one is a maintainer decision and neither is stale. green; `privacy:scan` passed. - Sync: merged `dev` (post-#1012, `088194a3a`) into the branch so the head is mergeable and CI can run; base sync of the stack continues with RI-07. + +### RI-07 - feat/ri-07-quota-aware-routing + +- Base SHA: `909ce21d4ffe4dbcad9c9baa2efb1c94e1e7dcd6` (RI-06 head) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 2 fixed pre-push - (1) `minQuotaHeadroom` was + missing from the schema REQUIRE_KEYS so normalization silently dropped it; + (2) `minQuotaHeadroom` was missing from `OcxRoutingProfileRequirements` + (types.ts) - typecheck caught it. +- Final commit: pending (recorded after commit) +- PR: pending +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/quota-scoring.test.ts`: 7/7 pass - codex-pool and + anthropic evidence, unknown-stays-unknown, unknown-quota policy + (exclude/penalize/allow), headroom preference, minQuotaHeadroom gating, + account-selection boundary + - Focused regression suites: 203/203 pass across 9 files + - `bun run privacy:scan`: passed +- Remaining Low findings: none + +### RI-07 review round (full-review #1014 + simplify) + +- Base SHA: `909ce21d4` (RI-06 head) -> rebased onto the reviewed RI-06 head + `a9c6f8e8` so the stack stays aligned; PR head `9dbdce2ab` before fixes. +- Simplify (approved): misindented `require.minContextWindow` check fixed in + `profile.ts`; shared RI-06 simplify carried in by the rebase. +- Bot-thread + own findings fixed in this PR: + 1. evaluator: `minQuotaHeadroom` now gates only KNOWN headroom; unknown + quota is governed by `unknownEvidence.quota` (labeled `unknown-quota`, + never `unknown-capability`); + 2. router: quota evidence receives the active codex account id so runtime + quota scoring reflects cached quota when a deterministic account exists; + 3. dry-run API: omitted `candidates` populate quota evidence alongside + capability + health; + 4. shared RI-06 review-round fixes (indexer tail, nested image evidence, + request-side requirements, alias namespace collision, policy fallthrough, + adapter tool inference) land here via the rebase. +- Verification: `tsc --noEmit` 0 errors; focused suites green (252/252 across + the routing set); `privacy:scan` passed. +- Base sync deferred: waiting for RI-05 (#1012) to merge before updating + these branches from `dev`. + +## Baseline note + +The full-suite baseline on this Windows machine did not complete within the +available window (background run, >3h, no summary emitted; the suite is +~8k tests and this machine is heavily loaded). Focused suites, typecheck and +privacy:scan pass per PR; the upstream PR #966 verification report records +~7941 pass / 10 environmental failures on clean dev. A final full-suite +attempt is scheduled at stack end. diff --git a/src/routing/profile.ts b/src/routing/profile.ts index 4f49c07ae..3f746372a 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -250,7 +250,7 @@ export function routingProfileIssues( issues.push({ path: ["require"], message: "require must be an object" }); } else { const require = body.require as Record; - if (require.minContextWindow !== undefined + if (require.minContextWindow !== undefined && (typeof require.minContextWindow !== "number" || !Number.isInteger(require.minContextWindow) || require.minContextWindow < 1)) { diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index b3fa6ec0b..b6ab51809 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -10,6 +10,7 @@ import { listRoutingProfileIds, getRoutingProfile, policyPublicModelId } from ". import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequestEvidence } from "../../routing/evaluator"; import { candidateCapabilityEvidence } from "../../routing/capability"; import { policyCandidateHealthEvidence } from "../../routing/health"; +import { quotaEvidenceForCandidate } from "../../routing/quota"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; @@ -112,6 +113,10 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis model: candidate.model, capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), health: policyCandidateHealthEvidence(config, candidate), + quota: quotaEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + }), })) : parseCandidateEvidence(body.candidates); if (candidateEvidence === null) { diff --git a/tests/quota-scoring.test.ts b/tests/quota-scoring.test.ts index 54eeb8329..d0ba03775 100644 --- a/tests/quota-scoring.test.ts +++ b/tests/quota-scoring.test.ts @@ -6,6 +6,8 @@ import { updateAccountQuota } from "../src/codex/quota"; import { setCachedProviderAccountQuotaForTests, clearAccountQuotaCache } from "../src/providers/quota"; import { quotaEvidenceForCandidate, quotaScore } from "../src/routing/quota"; import { evaluatePolicyProfile, QUOTA_UNKNOWN_PENALTY_SCORE } from "../src/routing/evaluator"; +import { routeModel } from "../src/router"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import type { OcxConfig } from "../src/types"; let testDir = ""; @@ -19,6 +21,7 @@ beforeEach(() => { afterEach(() => { clearAccountQuotaCache(); + closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (testDir) rmSync(testDir, { recursive: true, force: true }); @@ -143,6 +146,55 @@ describe("quota-aware scoring (RI-07)", () => { expect(enough.candidates[0]!.eligible).toBe(true); }); + test("minQuotaHeadroom unknown is governed by the quota policy, not capability", () => { + const strict = config({ + routingProfiles: { + g: { + candidates: [{ provider: "a", model: "m1" }], + require: { minQuotaHeadroom: 0.5 }, + unknownEvidence: { capability: "exclude", health: "penalize", quota: "exclude", cost: "penalize" }, + }, + }, + }); + // Unknown quota at execution: the quota "exclude" policy applies and is + // labeled `unknown-quota` - never the capability `unknown-capability` code. + const excluded = evaluatePolicyProfile(strict, "g", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(excluded.candidates[0]!.eligible).toBe(false); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-quota")).toBe(true); + expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-capability")).toBe(false); + + // Default "penalize": unknown headroom stays eligible with the quota floor. + const penalizing = config({ + routingProfiles: { + g: { + candidates: [{ provider: "a", model: "m1" }], + require: { minQuotaHeadroom: 0.5 }, + }, + }, + }); + const penalized = evaluatePolicyProfile(penalizing, "g", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(penalized.candidates[0]!.eligible).toBe(true); + expect(penalized.candidates[0]!.score!.components.quota).toBe(QUOTA_UNKNOWN_PENALTY_SCORE); + }); + + test("execution path passes the active codex account into quota evidence", async () => { + updateAccountQuota("pool-a", 30, 1_800_000_000_000, 20, 1_900_000_000_000); + const cfg = config({ + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + quotaRoute: { candidates: [{ provider: "openai", model: "gpt-5.6" }] }, + }, + }); + const route = routeModel(cfg, "policy/quotaRoute"); + expect(route.routeDecision!.candidates[0]!.quota?.known).toBe(true); + expect(route.routeDecision!.candidates[0]!.quota?.headroom).toBeCloseTo(0.7, 2); + }); + test("exact account selectors and pool strategies remain authoritative", () => { // Policy execution never invents account selection: candidates without // account refs get unknown quota evidence and the profile policy decides. From e74a211aaf2e8d46cf4757dd8d612ae13d0e5b7a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 06:18:56 +0200 Subject: [PATCH 3/8] docs(routing): document runtime unknown quota; add review regression tests (RI-07) --- .../docs/reference/configuration/routing.md | 6 +++ tests/policy-execution.test.ts | 18 +++++++++ tests/request-evidence.test.ts | 39 +++++++++++++++++++ tests/request-history-index.test.ts | 22 +++++++++++ 4 files changed, 85 insertions(+) create mode 100644 tests/request-evidence.test.ts diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index bc71a1852..3e3655526 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -152,6 +152,12 @@ CLI: `ocx route policy list [--json]`, `ocx route policy show [--json]`, an `ocx route policy dry-run [--model-context ] [--tools] [--image] [--structured-output] [--json]`. Dry-run evaluates candidates without sending any upstream request. +Quota evidence (`optimize.quota`, `require.minQuotaHeadroom`, `unknownEvidence.quota`) comes from +the local Codex pool and Anthropic account quota caches, which are keyed by account. Runtime policy +candidates carry no account reference, so their quota evidence is honestly unknown (`penalize` is +the default). To see quota-aware behavior, supply account-scoped evidence or account refs through +the dry-run/API candidate evidence (`candidates[].accountRef` / `candidates[].codexAccountId`). + ### Combos vs policy profiles - A **combo** is explicit ordered/weighted target routing and failover: the configured order (or diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index 77570fb96..2b0bb7975 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -114,6 +114,24 @@ describe("policy execution (RI-05)", () => { expect(route.modelId).toBe("m1"); }); + test("concrete selection never re-enters policy lookup (alias recursion guard)", () => { + // A profile whose winning candidate is a plain provider/model reference + // must not re-resolve policy aliases on the recursion, otherwise an alias + // that shadows the target would recurse until stack overflow. + const config = baseConfig({ + routingProfiles: { + direct: { + alias: "direct/a", + candidates: [{ provider: "a", model: "m1" }], + }, + }, + }); + const route = routeModel(config, "policy/direct"); + expect(route.routeKind).toBe("policy"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + }); + test("existing explicit, combo, native and default routes are unchanged", () => { const config = baseConfig(); expect(routeModel(config, "a/m1")).toMatchObject({ routeKind: "explicit-provider", providerName: "a", modelId: "m1" }); diff --git a/tests/request-evidence.test.ts b/tests/request-evidence.test.ts new file mode 100644 index 000000000..21213c209 --- /dev/null +++ b/tests/request-evidence.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from "bun:test"; +import { evidenceFromBody } from "../src/routing/request-evidence"; + +describe("request evidence extraction (RI-05)", () => { + test("top-level responses input image part is detected", () => { + const evidence = evidenceFromBody({ + input: [{ type: "input_image", image_url: "https://example.test/i.png" }], + }); + expect(evidence.imageInputRequired).toBe(true); + }); + + test("images nested under input[].content are detected", () => { + const evidence = evidenceFromBody({ + input: [{ type: "message", content: [{ type: "input_image", image_url: "https://example.test/i.png" }] }], + }); + expect(evidence.imageInputRequired).toBe(true); + }); + + test("images nested under messages[].content are detected (chat/claude bodies)", () => { + const evidence = evidenceFromBody({ + messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.test/i.png" } }] }], + }); + expect(evidence.imageInputRequired).toBe(true); + }); + + test("text-only bodies produce no image requirement", () => { + const evidence = evidenceFromBody({ + messages: [{ role: "user", content: "hello" }], + }); + expect(evidence.imageInputRequired).toBeUndefined(); + }); + + test("tools array produces toolsRequired", () => { + const evidence = evidenceFromBody({ + tools: [{ type: "function", function: { name: "x" } }], + }); + expect(evidence.toolsRequired).toBe(true); + }); +}); diff --git a/tests/request-history-index.test.ts b/tests/request-history-index.test.ts index 64d2abc1b..97d6a0a30 100644 --- a/tests/request-history-index.test.ts +++ b/tests/request-history-index.test.ts @@ -126,6 +126,28 @@ describe("request-history index (RI-02)", () => { expect(third.meta.lastError).toBe(""); }); + test("rows missing mandatory columns are skipped, not rejected", async () => { + for (const row of seedRows(3)) appendUsageEntry(row); + const { appendFileSync } = await import("node:fs"); + const { usageLogPath } = await import("../src/usage/log"); + // A complete row is indexable even with no usage details. + appendFileSync( + usageLogPath(), + JSON.stringify({ requestId: "lean", timestamp: 9000, provider: "a", model: "m1", status: 200, durationMs: 5 }) + "\n", + "utf-8", + ); + // A row missing mandatory NOT NULL columns (model/status/durationMs) must + // be skipped by the parser instead of throwing during the insert. + appendFileSync( + usageLogPath(), + JSON.stringify({ requestId: "broken", timestamp: 9001, provider: "a" }) + "\n", + "utf-8", + ); + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.some(row => row.requestId === "lean")).toBe(true); + expect(page.rows.some(row => row.requestId === "broken")).toBe(false); + }); + test("large history indexes fully and paginates without duplicates or misses", async () => { const rows = seedRows(1500, 10_000); for (const row of rows) appendUsageEntry(row); From 7aebbd36d7d09d724ce414fcae3728bbec2acb15 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:26:12 +0200 Subject: [PATCH 4/8] test(routing): update score totals for quota-weighted composite (RI-07) --- tests/policy-execution.test.ts | 4 ++-- tests/routing-profile.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts index 2b0bb7975..30386dd7d 100644 --- a/tests/policy-execution.test.ts +++ b/tests/policy-execution.test.ts @@ -102,8 +102,8 @@ describe("policy execution (RI-05)", () => { // RI-06: unknown health under the default "penalize" policy folds a // penalized health floor into the score. expect(trace.candidates[0]!.score).toMatchObject({ - total: 0.825, - components: { configuredPriority: 1, health: 0.3 }, + total: 0.755, + components: { configuredPriority: 1, health: 0.3, quota: 0.3 }, }); }); diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index c7ddc9e02..54973362f 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -318,8 +318,8 @@ describe("routing profiles (RI-04)", () => { // RI-06: unknown health under the default "penalize" policy folds a // penalized health floor into the score. expect(result.trace.candidates[0]!.score).toMatchObject({ - total: 0.825, - components: { configuredPriority: 1, health: 0.3 }, + total: 0.755, + components: { configuredPriority: 1, health: 0.3, quota: 0.3 }, }); }); From 480f1578a502a2d05455ca4f6bf070b1d8e103d2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:17:50 +0200 Subject: [PATCH 5/8] fix(routing): address review threads on quota evidence (RI-07) --- .../docs/reference/configuration/routing.md | 6 ++-- src/router.ts | 4 +++ src/routing/evaluator.ts | 12 +++++-- src/routing/quota.ts | 29 +++++++++++++++-- .../management/routing-profile-routes.ts | 20 ++++++++++++ tests/routing-profile.test.ts | 32 +++++++++++++++++++ 6 files changed, 97 insertions(+), 6 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 3e3655526..b994b5f58 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -155,8 +155,10 @@ Dry-run evaluates candidates without sending any upstream request. Quota evidence (`optimize.quota`, `require.minQuotaHeadroom`, `unknownEvidence.quota`) comes from the local Codex pool and Anthropic account quota caches, which are keyed by account. Runtime policy candidates carry no account reference, so their quota evidence is honestly unknown (`penalize` is -the default). To see quota-aware behavior, supply account-scoped evidence or account refs through -the dry-run/API candidate evidence (`candidates[].accountRef` / `candidates[].codexAccountId`). +the default). To see quota-aware behavior in a dry-run, supply account refs through the dry-run/API +candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai`) or +`candidates[].accountRef` (Anthropic) derives the matching cached account quota; an explicit +`candidates[].quota` object is echoed as given. ### Combos vs policy profiles diff --git a/src/router.ts b/src/router.ts index a8910b8bf..b8dcd36c9 100644 --- a/src/router.ts +++ b/src/router.ts @@ -512,6 +512,10 @@ function routeModelInternal( provider: candidate.provider, model: candidate.model, codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + && providerCodexAccountMode( + OPENAI_CODEX_PROVIDER_ID, + config.providers[OPENAI_CODEX_PROVIDER_ID], + ) === "pool" ? getEffectiveActiveCodexAccountId(config) : undefined, }), diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 6b97e0a70..04fdf01f9 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -46,6 +46,8 @@ export interface PolicyCandidateEvidence { provider: string; model: string; accountRef?: string; + /** Codex pool account id (provider "openai"); used to derive account-scoped quota evidence. */ + codexAccountId?: string; capability?: RouteCapabilityEvidence; health?: RouteHealthEvidence; quota?: RouteQuotaEvidence; @@ -338,8 +340,14 @@ export function evaluatePolicyProfile( const priorityScore = configuredPriorityScore(index, profile.candidates.length); const healthWeight = profile.optimize.health; const quotaWeight = profile.optimize.quota; - const implementedWeight = healthWeight + quotaWeight; - const priorityWeight = Math.max(0, 1 - implementedWeight); + // Only spend a dimension's weight when a value is actually present: + // "allow" leaves missing health/quota components null, so subtracting + // their weights would shrink the priority share for evidence the profile + // explicitly permits to be absent. Renormalize those weights back into + // priority instead of silently changing the ranking semantics. + const spentHealth = healthValue !== null ? healthWeight : 0; + const spentQuota = quotaValue !== null ? quotaWeight : 0; + const priorityWeight = Math.max(0, 1 - spentHealth - spentQuota); const components: RouteScoreEvidence["components"] = { configuredPriority: priorityScore }; let total = priorityWeight * priorityScore; if (healthWeight > 0 && healthValue !== null) { diff --git a/src/routing/quota.ts b/src/routing/quota.ts index d729d53f7..327c6abd9 100644 --- a/src/routing/quota.ts +++ b/src/routing/quota.ts @@ -54,10 +54,15 @@ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuota if (input.provider === "anthropic" && input.accountRef) { const quota = getCachedProviderAccountQuota("anthropic", input.accountRef); if (quota) { - const percents = [quota.fiveHourPercent, quota.weeklyPercent, quota.monthlyPercent] + // Anthropic per-family buckets (e.g. Opus / Sonnet) are stricter than the + // broad account windows: fold the candidate model's matching bucket into + // headroom and exhaustion so a model-specific overage is not hidden by + // a healthy aggregate window. + const family = anthropicFamilyWindow(input.model, quota.customWindows ?? []); + const percents = [quota.fiveHourPercent, quota.weeklyPercent, quota.monthlyPercent, family?.percent] .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; - const resets = [quota.fiveHourResetAt, quota.weeklyResetAt, quota.monthlyResetAt] + const resets = [quota.fiveHourResetAt, quota.weeklyResetAt, quota.monthlyResetAt, family?.resetAt] .filter((value): value is number => typeof value === "number" && Number.isFinite(value)) .filter(value => value > Date.now()); return { @@ -75,6 +80,26 @@ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuota return { known: false }; } +/** + * Match the candidate model to an Anthropic per-family quota window. Window + * labels from the provider probe are "Opus" / "Sonnet"; model ids carry the + * family as a segment (e.g. `claude-opus-...`, `claude-sonnet-...`). Returns + * undefined when no family window is cached or no label matches. + */ +function anthropicFamilyWindow( + model: string, + windows: Array<{ label: string; percent?: number; resetAt?: number }>, +): { percent?: number; resetAt?: number } | undefined { + const normalized = model.toLowerCase(); + for (const window of windows) { + const family = window.label.trim().toLowerCase(); + if (family && normalized.includes(family)) { + return window; + } + } + return undefined; +} + /** * Deterministic quota score in [0,1]: larger available headroom scores * higher; exhausted evidence scores 0. Unknown evidence returns null so the diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index b6ab51809..b515db6b8 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -11,6 +11,9 @@ import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequest import { candidateCapabilityEvidence } from "../../routing/capability"; import { policyCandidateHealthEvidence } from "../../routing/health"; import { quotaEvidenceForCandidate } from "../../routing/quota"; +import { providerCodexAccountMode } from "../../providers/registry"; +import { getEffectiveActiveCodexAccountId } from "../../codex/routing"; +import { OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; @@ -60,6 +63,7 @@ function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null provider, model, ...(typeof item.accountRef === "string" ? { accountRef: item.accountRef } : {}), + ...(typeof item.codexAccountId === "string" ? { codexAccountId: item.codexAccountId } : {}), // Dry-run evidence is caller-supplied and echoed back in the result as // given; the trace's candidate rows carry only score/exclusions, which // the trace builder bounds. Structural casts keep the API permissive. @@ -67,6 +71,15 @@ function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null ...(isPlainRecord(item.health) ? { health: item.health as unknown as PolicyCandidateEvidence["health"] } : {}), ...(isPlainRecord(item.quota) ? { quota: item.quota as unknown as PolicyCandidateEvidence["quota"] } : {}), ...(isPlainRecord(item.cost) ? { cost: item.cost as unknown as PolicyCandidateEvidence["cost"] } : {}), + // Derive account-scoped quota evidence from the documented refs when the + // caller does not supply an explicit quota object, so a dry-run following + // the documented shape reports the same cached account quota as routing. + ...(item.quota === undefined && typeof item.codexAccountId === "string" + ? { quota: quotaEvidenceForCandidate({ provider, model, codexAccountId: item.codexAccountId }) } + : {}), + ...(item.quota === undefined && typeof item.accountRef === "string" && typeof item.codexAccountId !== "string" + ? { quota: quotaEvidenceForCandidate({ provider, model, accountRef: item.accountRef }) } + : {}), }); } return out; @@ -116,6 +129,13 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis quota: quotaEvidenceForCandidate({ provider: candidate.provider, model: candidate.model, + codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + && providerCodexAccountMode( + OPENAI_CODEX_PROVIDER_ID, + config.providers[OPENAI_CODEX_PROVIDER_ID], + ) === "pool" + ? getEffectiveActiveCodexAccountId(config) + : undefined, }), })) : parseCandidateEvidence(body.candidates); diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index 54973362f..148ca4d5f 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -3,6 +3,8 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { validateConfigCandidate } from "../src/config"; +import { updateAccountQuota } from "../src/codex/quota"; +import { clearAccountQuotaCache } from "../src/providers/quota"; import { handleManagementAPI } from "../src/server/management-api"; import { ManagementRequest } from "./helpers/management-auth"; import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; @@ -28,6 +30,7 @@ beforeEach(() => { }); afterEach(() => { + clearAccountQuotaCache(); closeRequestHistoryIndex(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; @@ -436,4 +439,33 @@ describe("routing profiles (RI-04)", () => { expect(body.candidates?.[0]?.eligible).toBe(false); expect(body.candidates?.[0]?.exclusions?.some(exclusion => exclusion.code === "cooldown")).toBe(true); }); + + test("API dry-run derives quota evidence from candidates[].codexAccountId", async () => { + updateAccountQuota("pool-a", 30, 1_800_000_000_000, 20, 1_900_000_000_000); + const config = baseConfig({ + providers: { + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + routingProfiles: { + only: { candidates: [{ provider: "openai", model: "gpt-5.6" }] }, + }, + }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "only", + evidence: {}, + candidates: [{ provider: "openai", model: "gpt-5.6", codexAccountId: "pool-a" }], + }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { candidates?: Array<{ quota?: { known?: boolean; headroom?: number } }> }; + // The documented candidates[].codexAccountId ref derives the cached pool + // quota (30% weekly / 20% monthly => 0.7 headroom) instead of unknown. + expect(body.candidates?.[0]?.quota?.known).toBe(true); + expect(body.candidates?.[0]?.quota?.headroom).toBeCloseTo(0.7, 2); + }); }); From 850c62531ad5802cfe794d825e390a310ab335eb Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:29:58 +0200 Subject: [PATCH 6/8] fix(routing): address CodeRabbit review on quota evidence (RI-07) --- .../001_pr_stack_status.md | 9 ++++--- .../docs/reference/configuration/routing.md | 14 +++++++---- src/router.ts | 16 ++++++++++-- src/routing/evaluator.ts | 25 +++++++++++-------- src/routing/quota.ts | 23 ++++++++++++----- .../management/routing-profile-routes.ts | 16 ++++++++++-- tests/quota-scoring.test.ts | 14 +++++++++++ 7 files changed, 88 insertions(+), 29 deletions(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 4de69ce8f..bf50bcb4d 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -223,14 +223,15 @@ other; closing one is a maintainer decision and neither is stale. missing from the schema REQUIRE_KEYS so normalization silently dropped it; (2) `minQuotaHeadroom` was missing from `OcxRoutingProfileRequirements` (types.ts) - typecheck caught it. -- Final commit: pending (recorded after commit) -- PR: pending +- Final commit: `480f1578` (review-thread fixes; earlier commits `b562fdb9`, + `288dd8a9`, `e74a211a`, `7aebbd36`) +- PR: #1014 - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) - - `bun run test tests/quota-scoring.test.ts`: 7/7 pass - codex-pool and + - `bun run test tests/quota-scoring.test.ts`: 9/9 pass - codex-pool and anthropic evidence, unknown-stays-unknown, unknown-quota policy (exclude/penalize/allow), headroom preference, minQuotaHeadroom gating, - account-selection boundary + account-selection boundary, plan-aware window selection - Focused regression suites: 203/203 pass across 9 files - `bun run privacy:scan`: passed - Remaining Low findings: none diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index b994b5f58..66ecba415 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -153,9 +153,12 @@ CLI: `ocx route policy list [--json]`, `ocx route policy show [--json]`, an Dry-run evaluates candidates without sending any upstream request. Quota evidence (`optimize.quota`, `require.minQuotaHeadroom`, `unknownEvidence.quota`) comes from -the local Codex pool and Anthropic account quota caches, which are keyed by account. Runtime policy -candidates carry no account reference, so their quota evidence is honestly unknown (`penalize` is -the default). To see quota-aware behavior in a dry-run, supply account refs through the dry-run/API +the local Codex pool and Anthropic account quota caches, which are keyed by account. In **Pool** mode +the canonical `openai` provider preserves its existing account selection, then reads quota for the +selected account; **Direct** mode reads quota only from the current (caller/main) account. For other +providers (e.g. Anthropic), runtime candidates use the provider's active account. Quota evidence +never changes account selection, session affinity, cooldowns, or switching behavior — it only feeds +policy scoring. To see quota-aware behavior in a dry-run, supply account refs through the dry-run/API candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai`) or `candidates[].accountRef` (Anthropic) derives the matching cached account quota; an explicit `candidates[].quota` object is echoed as given. @@ -168,8 +171,9 @@ candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai` requirements filter first, then deterministic scoring ranks the survivors. Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate -is chosen. Profile scoring currently uses the configured-priority component only; health (RI-06), -quota (RI-07), and cost (RI-08) score dimensions are planned. Per-request route-decision traces are +is chosen. Profile scoring combines the configured-priority component with the health (RI-06) and +quota (RI-07) score dimensions where evidence is present; cost participates through the +`limits.maxEstimatedCostUsd` cap rather than a score weight. Per-request route-decision traces are recorded when a policy profile executes. ### Catalog eligibility diff --git a/src/router.ts b/src/router.ts index b8dcd36c9..873271d40 100644 --- a/src/router.ts +++ b/src/router.ts @@ -23,6 +23,7 @@ import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; import { getEffectiveActiveCodexAccountId } from "./codex/routing"; +import { getAccountSet } from "./oauth/store"; import { buildRouteDecisionTrace, type RouteDecisionKind, @@ -511,12 +512,23 @@ function routeModelInternal( quota: quotaEvidenceForCandidate({ provider: candidate.provider, model: candidate.model, - codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID && providerCodexAccountMode( OPENAI_CODEX_PROVIDER_ID, config.providers[OPENAI_CODEX_PROVIDER_ID], ) === "pool" - ? getEffectiveActiveCodexAccountId(config) + ? (() => { + const codexAccountId = getEffectiveActiveCodexAccountId(config); + return { + codexAccountId, + codexAccountPlan: codexAccountId + ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan + : undefined, + }; + })() + : {}), + accountRef: candidate.provider === "anthropic" + ? getAccountSet("anthropic")?.activeAccountId : undefined, }), })); diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 04fdf01f9..6f427713d 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -114,16 +114,21 @@ function requirementFor( requirements.push({ id: "min-context-window", expected: require.minContextWindow, outcome: "unknown" }); } } - // minQuotaHeadroom gates only KNOWN headroom. Unknown quota is governed by - // the profile's `unknownEvidence.quota` policy (exclude / penalize / allow) - // via the quota score path - never by the capability unknown policy. - if (require.minQuotaHeadroom !== undefined && typeof quota?.headroom === "number") { - requirements.push({ - id: "min-quota-headroom", - expected: require.minQuotaHeadroom, - actual: quota.headroom, - outcome: quota.headroom >= require.minQuotaHeadroom ? "satisfied" : "unsatisfied", - }); + // minQuotaHeadroom gates only KNOWN quota via the normalized score, which + // maps exhaustion to zero and unknown/incomplete evidence to null. Unknown + // quota is governed by the profile's `unknownEvidence.quota` policy + // (exclude / penalize / allow) via the quota score path - never by the + // capability unknown policy. + if (require.minQuotaHeadroom !== undefined) { + const quotaHeadroom = quotaScore(quota); + if (quotaHeadroom !== null) { + requirements.push({ + id: "min-quota-headroom", + expected: require.minQuotaHeadroom, + actual: quotaHeadroom, + outcome: quotaHeadroom >= require.minQuotaHeadroom ? "satisfied" : "unsatisfied", + }); + } } const tools = booleanRequirement("tools", require.tools, capability?.tools); if (tools) requirements.push(tools); diff --git a/src/routing/quota.ts b/src/routing/quota.ts index 327c6abd9..06cc8eecc 100644 --- a/src/routing/quota.ts +++ b/src/routing/quota.ts @@ -12,7 +12,7 @@ * carries an account reference (dry-run/evaluate), never invented. */ -import { getAccountQuota, isCodexQuotaExhausted } from "../codex/quota"; +import { codexQuotaWindowForPlan, getAccountQuota, isCodexQuotaExhausted } from "../codex/quota"; import { getCachedProviderAccountQuota } from "../providers/quota"; import type { RouteQuotaEvidence } from "./trace"; @@ -23,6 +23,8 @@ export interface QuotaEvidenceInput { accountRef?: string; /** Codex pool account id (provider "openai"). */ codexAccountId?: string; + /** The account's plan (provider "openai"); selects the governing quota window. */ + codexAccountPlan?: string; } /** @@ -33,18 +35,27 @@ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuota if (input.provider === "openai" && input.codexAccountId) { const quota = getAccountQuota(input.codexAccountId); if (quota) { - const percents = [quota.weeklyPercent, quota.monthlyPercent] - .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + // Go/Free accounts report a 30-day window only; weekly windows gate + // everything else. `codexQuotaWindowForPlan` is the single shared rule + // (parser, exhaustion, recovery), so select the plan-specific bars here + // too instead of always combining weekly + monthly. + const monthly = codexQuotaWindowForPlan(input.codexAccountPlan) === "monthly"; + const percents = [ + ...(monthly ? [] : [quota.weeklyPercent]), + quota.monthlyPercent, + ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)); const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; - const resets = [quota.weeklyResetAt, quota.monthlyResetAt] - .filter((value): value is number => typeof value === "number" && Number.isFinite(value)) + const resets = [ + ...(monthly ? [] : [quota.weeklyResetAt]), + quota.monthlyResetAt, + ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)) .filter(value => value > Date.now()); return { known: true, ...(maxPercent !== undefined ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } : {}), - exhausted: isCodexQuotaExhausted(quota), + exhausted: isCodexQuotaExhausted(quota, input.codexAccountPlan), ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), source: "codex-pool", }; diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index b515db6b8..ebfa491d9 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -13,6 +13,7 @@ import { policyCandidateHealthEvidence } from "../../routing/health"; import { quotaEvidenceForCandidate } from "../../routing/quota"; import { providerCodexAccountMode } from "../../providers/registry"; import { getEffectiveActiveCodexAccountId } from "../../codex/routing"; +import { getAccountSet } from "../../oauth/store"; import { OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; @@ -129,12 +130,23 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis quota: quotaEvidenceForCandidate({ provider: candidate.provider, model: candidate.model, - codexAccountId: candidate.provider === OPENAI_CODEX_PROVIDER_ID + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID && providerCodexAccountMode( OPENAI_CODEX_PROVIDER_ID, config.providers[OPENAI_CODEX_PROVIDER_ID], ) === "pool" - ? getEffectiveActiveCodexAccountId(config) + ? (() => { + const codexAccountId = getEffectiveActiveCodexAccountId(config); + return { + codexAccountId, + codexAccountPlan: codexAccountId + ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan + : undefined, + }; + })() + : {}), + accountRef: candidate.provider === "anthropic" + ? getAccountSet("anthropic")?.activeAccountId : undefined, }), })) diff --git a/tests/quota-scoring.test.ts b/tests/quota-scoring.test.ts index d0ba03775..30a637d84 100644 --- a/tests/quota-scoring.test.ts +++ b/tests/quota-scoring.test.ts @@ -112,6 +112,20 @@ describe("quota-aware scoring (RI-07)", () => { ]); expect(penalized.candidates[0]!.eligible).toBe(true); expect(penalized.candidates[0]!.score!.components.quota).toBe(QUOTA_UNKNOWN_PENALTY_SCORE); + + const allowing = config({ + routingProfiles: { + q: { + candidates: [{ provider: "a", model: "m1" }], + unknownEvidence: { capability: "allow", health: "penalize", quota: "allow", cost: "penalize" }, + }, + }, + }); + const allowed = evaluatePolicyProfile(allowing, "q", {}, [ + { provider: "a", model: "m1", capability: { contextWindow: 200000 } }, + ]); + expect(allowed.candidates[0]!.eligible).toBe(true); + expect(allowed.candidates[0]!.score!.components.quota).toBeUndefined(); }); test("larger headroom is preferred when quota scoring is weighted", () => { From 3fd1a00a53b89463a4f13e6e037282bce2c511cc Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:46:47 +0200 Subject: [PATCH 7/8] docs(routing): drop cost weight from profile scoring docs (RI-07) --- docs-site/src/content/docs/reference/configuration/routing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 66ecba415..7ca4b906a 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -104,7 +104,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `candidates` | `{ provider: string; model: string }[]` | required | Explicit allowlist of `provider/model` refs. No implicit expansion. | | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | -| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights; normalized deterministically. | +| `optimize?` | object | latency 0.55, health 0.25, quota 0.10 | Scoring weights; normalized deterministically. `cost` is accepted for schema compatibility but does not affect scoring — cost is enforced only through `limits.maxEstimatedCostUsd`. | | `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | @@ -135,7 +135,7 @@ candidate evidence is provided through the API (`POST /api/routing-profiles/dry- { "provider": "openai", "model": "gpt-5.6-sol" } ], "require": { "tools": true, "minContextWindow": 128000 }, - "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, + "optimize": { "latency": 0.55, "health": 0.25, "quota": 0.10 }, "limits": { "maxEstimatedCostUsd": 0.50 }, "unknownEvidence": { "capability": "exclude", From f1ec2d66d074f6a0f10d02431806793d59ca5cc7 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:59:17 +0200 Subject: [PATCH 8/8] docs(routing): document actual optimize scoring semantics (RI-07) --- .../content/docs/reference/configuration/routing.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 7ca4b906a..318edd275 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -104,7 +104,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `candidates` | `{ provider: string; model: string }[]` | required | Explicit allowlist of `provider/model` refs. No implicit expansion. | | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | -| `optimize?` | object | latency 0.55, health 0.25, quota 0.10 | Scoring weights; normalized deterministically. `cost` is accepted for schema compatibility but does not affect scoring — cost is enforced only through `limits.maxEstimatedCostUsd`. | +| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. Only `health` and `quota` have score dimensions; the configured-priority share is `1 - health - quota` (default 0.65), and `latency`/`cost` fold into that priority share rather than scoring independently. | | `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | @@ -135,7 +135,7 @@ candidate evidence is provided through the API (`POST /api/routing-profiles/dry- { "provider": "openai", "model": "gpt-5.6-sol" } ], "require": { "tools": true, "minContextWindow": 128000 }, - "optimize": { "latency": 0.55, "health": 0.25, "quota": 0.10 }, + "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, "limits": { "maxEstimatedCostUsd": 0.50 }, "unknownEvidence": { "capability": "exclude", @@ -172,9 +172,10 @@ candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai` Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate is chosen. Profile scoring combines the configured-priority component with the health (RI-06) and -quota (RI-07) score dimensions where evidence is present; cost participates through the -`limits.maxEstimatedCostUsd` cap rather than a score weight. Per-request route-decision traces are -recorded when a policy profile executes. +quota (RI-07) score dimensions where evidence is present; the `latency` and `cost` weights fold into +the priority share rather than scoring independently, and cost is also enforced through the +`limits.maxEstimatedCostUsd` cap. Per-request route-decision traces are recorded when a policy +profile executes. ### Catalog eligibility