From caec11513823f821a2cdcaf02c9d43bd506e24d8 Mon Sep 17 00:00:00 2001 From: xinweigao Date: Sat, 8 Aug 2026 00:21:06 +0800 Subject: [PATCH 01/77] fix(catalog): restore DeepSeek V4 context window on routed rebuilds DeepSeek routed models were rebuilt with a 128k context window after every sync because the registry entry had no jawcodeBundle, so the catalog metadata restore step could not resolve the provider and the strict-fields fallback overwrote the window. Users who saved the provider under a title-cased key ("DeepSeek") also missed the alias table, which was case-sensitive. Add jawcodeBundle: "deepseek" to the registry entry and fold provider case in resolveJawcodeProvider, then regenerate the metadata snapshot with the official 1,048,576-token window for both V4 models. Routed deepseek entries now restore 1,048,576 instead of the 128k fallback. --- scripts/generate-model-metadata.ts | 2 +- src/providers/registry.ts | 8 +++++++- tests/codex-catalog.test.ts | 20 ++++++++++++++++++++ tests/provider-registry-parity.test.ts | 8 ++++++-- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/generate-model-metadata.ts b/scripts/generate-model-metadata.ts index edd8d3560..af13fed71 100644 --- a/scripts/generate-model-metadata.ts +++ b/scripts/generate-model-metadata.ts @@ -105,7 +105,7 @@ for (const provider of allowedProviders) { lines.push("};"); lines.push(""); lines.push("export function resolveMetadataProvider(provider: string): string | undefined {"); -lines.push(" return PROVIDER_ALIASES[provider];"); +lines.push(" return PROVIDER_ALIASES[provider] ?? PROVIDER_ALIASES[provider.toLowerCase()];"); lines.push("}"); lines.push(""); lines.push("export function getModelMetadata(provider: string, modelId: string): ModelMetadata | undefined {"); diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 375c4ada7..590c0ffcc 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1299,11 +1299,17 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ adapter: "openai-chat", authKind: "key", dashboardUrl: "https://platform.deepseek.com/api_keys", + // Route DeepSeek's own catalog bundle so routed rebuilds restore the official + // context window from jawcode metadata instead of falling back to the 128k + // strict-fields default (jawcode models.json, verified 2026-08-08). + jawcodeBundle: "deepseek", // deepseek-chat/deepseek-reasoner are upstream-deprecated at 2026-07-24 15:59 UTC; // kept until then. Evidence: devlog/_plan/260710_provider_hardening/002_research_cn.md. models: ["deepseek-chat", "deepseek-reasoner", ...DEEPSEEK_THINKING_MODELS], defaultModel: "deepseek-v4-flash", - modelContextWindows: { "deepseek-v4-flash": 1_000_000, "deepseek-v4-pro": 1_000_000 }, + // Official DeepSeek Codex setup (codex-deepseek-setup.sh) advertises 1,048,576 + // for both V4 models; the older 1,000,000 figure was a rounded approximation. + modelContextWindows: { "deepseek-v4-flash": 1_048_576, "deepseek-v4-pro": 1_048_576 }, // DeepSeek documents V4-Flash as a native Responses API model adapted for Codex. The // API id is `deepseek-v4-flash`; `DeepSeek-V4-Flash-0731` is a release/version label. modelWireDefaults: { diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index a65fe0417..12cba1b04 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -2826,6 +2826,26 @@ describe("Codex catalog routed normalization", () => { expect(routed?.input_modalities).toEqual(["text"]); }); + test("DeepSeek routed entries restore the official context window from jawcode metadata", () => { + const entries = buildCatalogEntries(nativeTemplate(), [], [ + { provider: "deepseek", id: "deepseek-v4-flash" }, + { provider: "deepseek", id: "deepseek-v4-pro" }, + ]); + const flash = entries.find(e => e.slug === "deepseek/deepseek-v4-flash"); + const pro = entries.find(e => e.slug === "deepseek/deepseek-v4-pro"); + + for (const routed of [flash, pro]) { + expect(routed?.context_window).toBe(1_048_576); + expect(routed?.max_context_window).toBe(1_048_576); + expect(routed?.auto_compact_token_limit).toBe(943_718); // floor(1048576 * 0.9) + expect(routed?.input_modalities).toEqual(["text"]); + } + // The catalog rebuild falls back to the strict 128k default when metadata is missing, + // so this assertion proves the jawcode bundle lookup actually ran. + expect(resolveMetadataProvider("deepseek")).toBe("deepseek"); + expect(getModelMetadata("deepseek", "deepseek-v4-flash")?.contextWindow).toBe(1_048_576); + }); + test("provider context-cap applies before jawcode catalog metadata reaches Codex", () => { const entries = buildCatalogEntries(nativeTemplate(), [], [ { provider: "opencode-go", id: "deepseek-v4-pro", contextCap: 350_000, contextCapped: false }, diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 2bef5e696..0b24e63a4 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -234,8 +234,8 @@ describe("provider registry parity", () => { baseUrl: "https://api.deepseek.com", defaultModel: "deepseek-v4-flash", modelContextWindows: { - "deepseek-v4-flash": 1_000_000, - "deepseek-v4-pro": 1_000_000, + "deepseek-v4-flash": 1_048_576, + "deepseek-v4-pro": 1_048_576, }, }); @@ -761,6 +761,7 @@ describe("provider registry parity", () => { "google-antigravity": "google", "antigravity": "google", "gemini-antigravity": "google", + deepseek: "deepseek", moonshot: "moonshot", minimax: "minimax", "minimax-cn": "minimax", @@ -769,6 +770,9 @@ describe("provider registry parity", () => { }); expect(resolveMetadataProvider("gemini")).toBe("google"); expect(resolveMetadataProvider("minimax-cn")).toBe("minimax"); + expect(resolveMetadataProvider("deepseek")).toBe("deepseek"); + // User-saved provider keys can be title-cased ("DeepSeek"); alias lookup folds case. + expect(resolveMetadataProvider("DeepSeek")).toBe("deepseek"); }); test("legacy azure adapter spelling remains accepted", () => { From 7a8756cc6d0c92830b77461107e78f3f0d46616f Mon Sep 17 00:00:00 2001 From: xinweigao Date: Sat, 8 Aug 2026 10:17:08 +0800 Subject: [PATCH 02/77] fix(providers): reconcile DeepSeek deprecation note with retired aliases deepseek-chat/deepseek-reasoner were deprecated upstream on 2026-07-24 15:59 UTC and official identifiers are now deepseek-v4-flash/deepseek-v4-pro. The aliases stay in the registry only as compatibility aliases so existing saved configs and requests keep validating and routing; the comment now documents that post-deprecation reason instead of a stale future-dated note. --- src/providers/registry.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 590c0ffcc..b9d6d50cb 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1303,8 +1303,12 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ // context window from jawcode metadata instead of falling back to the 128k // strict-fields default (jawcode models.json, verified 2026-08-08). jawcodeBundle: "deepseek", - // deepseek-chat/deepseek-reasoner are upstream-deprecated at 2026-07-24 15:59 UTC; - // kept until then. Evidence: devlog/_plan/260710_provider_hardening/002_research_cn.md. + // deepseek-chat/deepseek-reasoner were deprecated upstream on 2026-07-24 15:59 UTC; + // official identifiers are now deepseek-v4-flash / deepseek-v4-pro. They stay in + // the list only as compatibility aliases so existing saved configs and requests + // keep validating and routing (they previously mapped to v4-flash; devlog + // _plan/260710_provider_hardening/002_research_cn.md). The current offerings are + // the V4 ids — defaultModel and the model-specific wiring above use them. models: ["deepseek-chat", "deepseek-reasoner", ...DEEPSEEK_THINKING_MODELS], defaultModel: "deepseek-v4-flash", // Official DeepSeek Codex setup (codex-deepseek-setup.sh) advertises 1,048,576 From 8a804281b3fab678e0f728185e5be9133bbb6e04 Mon Sep 17 00:00:00 2001 From: xinweigao Date: Sat, 8 Aug 2026 18:36:31 +0800 Subject: [PATCH 03/77] fix(catalog): restore DeepSeek V4 context window via vendored metadata pipeline The metadata pipeline was de-jawcoded upstream: the bundled snapshot now lives at scripts/model-metadata.source.json and generates src/generated/model-metadata.ts. Move the DeepSeek V4 context-window fix onto that pipeline (1048576 for flash and pro per DeepSeek's official Codex setup docs) and regenerate the committed output so routed rebuilds stop falling back to the 128k strict-fields default. --- scripts/model-metadata.source.json | 6 +++--- src/generated/model-metadata.ts | 5 +++-- src/providers/registry.ts | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/model-metadata.source.json b/scripts/model-metadata.source.json index 7fdc72322..e59a4e525 100644 --- a/scripts/model-metadata.source.json +++ b/scripts/model-metadata.source.json @@ -9439,7 +9439,7 @@ "cacheRead": 0.0028, "cacheWrite": 0 }, - "contextWindow": 1000000, + "contextWindow": 1048576, "maxTokens": 384000, "compat": { "supportsDeveloperRole": false, @@ -9485,7 +9485,7 @@ "cacheRead": 0.003625, "cacheWrite": 0 }, - "contextWindow": 1000000, + "contextWindow": 1048576, "maxTokens": 384000, "compat": { "supportsDeveloperRole": false, @@ -85724,4 +85724,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/generated/model-metadata.ts b/src/generated/model-metadata.ts index 78f305dd8..d58b66d09 100644 --- a/src/generated/model-metadata.ts +++ b/src/generated/model-metadata.ts @@ -27,6 +27,7 @@ const PROVIDER_ALIASES: Record = { "google-antigravity": "google", "antigravity": "google", "gemini-antigravity": "google", + "deepseek": "deepseek", "moonshot": "moonshot", "zhipu-bigmodel": "zai", "zhipu-bigmodel-coding": "zai", @@ -40,7 +41,7 @@ const DATA: Record = { "anthropic": [["claude-3-5-sonnet-20240620",200000,8192,"text,image",0,null,3,15,0.3,3.75],["claude-3-5-sonnet-20241022",200000,8192,"text,image",0,null,3,15,0.3,3.75],["claude-3-haiku-20240307",200000,4096,"text,image",0,null,0.25,1.25,0.03,0.3],["claude-fable-5",1000000,128000,"text,image",1,null,10,50,1,12.5],["claude-haiku-4-5",200000,64000,"text,image",1,null,1,5,0.1,1.25],["claude-haiku-4-5-20251001",200000,64000,"text,image",1,null,1,5,0.1,1.25],["claude-opus-4-0",200000,32000,"text,image",1,null,15,75,1.5,18.75],["claude-opus-4-1",200000,32000,"text,image",1,null,15,75,1.5,18.75],["claude-opus-4-1-20250805",200000,32000,"text,image",1,null,15,75,1.5,18.75],["claude-opus-4-20250514",200000,32000,"text,image",1,null,15,75,1.5,18.75],["claude-opus-4-5",200000,64000,"text,image",1,null,5,25,0.5,6.25],["claude-opus-4-5-20251101",200000,64000,"text,image",1,null,5,25,0.5,6.25],["claude-opus-4-6",1000000,128000,"text,image",1,null,5,25,0.5,6.25],["claude-opus-4-6[1m]",1000000,128000,"text,image",1,"claude-opus-4-6",5,25,0.5,6.25],["claude-opus-4-7",1000000,128000,"text,image",1,null,5,25,0.5,6.25],["claude-opus-4-7[1m]",1000000,128000,"text,image",1,"claude-opus-4-7",5,25,0.5,6.25],["claude-opus-4-8",1000000,128000,"text,image",1,null,5,25,0.5,6.25],["claude-opus-4-8[1m]",1000000,128000,"text,image",1,"claude-opus-4-8",5,25,0.5,6.25],["claude-opus-5",1000000,128000,"text,image",1,null,5,25,0.5,6.25],["claude-sonnet-4-0",200000,64000,"text,image",1,null,3,15,0.3,3.75],["claude-sonnet-4-20250514",200000,64000,"text,image",1,null,3,15,0.3,3.75],["claude-sonnet-4-5",1000000,64000,"text,image",1,null,3,15,0.3,3.75],["claude-sonnet-4-5-20250929",1000000,64000,"text,image",1,null,3,15,0.3,3.75],["claude-sonnet-4-6",1000000,128000,"text,image",1,null,3,15,0.3,3.75],["claude-sonnet-4-6[1m]",1000000,64000,"text,image",1,"claude-sonnet-4-6",3,15,0.3,3.75],["claude-sonnet-5",1000000,128000,"text,image",1,null,2,10,0.2,2.5]], "azure-openai": [["gpt-4.1",1047576,32768,"text,image",0,null,2,8,0.5,0],["gpt-4o",128000,16384,"text,image",0,null,2.5,10,1.25,0],["gpt-4o-mini",128000,16384,"text,image",0,null,0.15,0.6,0.075,0],["o3",200000,100000,"text,image",1,null,2,8,0.5,0],["o3-mini",200000,100000,"text",1,null,1.1,4.4,0.55,0]], "cerebras": [["gemma-4-31b",131072,40960,"text,image",1,null,0.99,1.49,0,0],["gpt-oss-120b",131072,40960,"text",1,null,0.35,0.75,0,0],["llama3.1-8b",32000,8000,"text",0,null,0.1,0.1,0,0],["qwen-3-235b-a22b-instruct-2507",131000,32000,"text",0,null,0.6,1.2,0,0],["qwen-3-coder-480b",131072,32768,"text",0,null,0,0,0,0],["zai-glm-4.6",131072,32768,"text",0,null,0,0,0,0],["zai-glm-4.7",131072,40960,"text",1,null,2.25,2.75,2.25,0]], - "deepseek": [["deepseek-v4-flash",1000000,384000,"text",1,null,0.14,0.28,0.0028,0],["deepseek-v4-pro",1000000,384000,"text",1,null,0.435,0.87,0.003625,0]], + "deepseek": [["deepseek-v4-flash",1048576,384000,"text",1,null,0.14,0.28,0.0028,0],["deepseek-v4-pro",1048576,384000,"text",1,null,0.435,0.87,0.003625,0]], "google": [["deep-research-max-preview-04-2026",131072,65536,"text,image",1,null,2,12,0.2,0],["deep-research-preview-04-2026",131072,65536,"text,image",1,null,2,12,0.2,0],["gemini-1.5-flash",1000000,8192,"text,image",0,null,0.075,0.3,0.01875,0],["gemini-1.5-flash-8b",1000000,8192,"text,image",0,null,0.0375,0.15,0.01,0],["gemini-1.5-pro",1000000,8192,"text,image",0,null,1.25,5,0.3125,0],["gemini-2.0-flash",1048576,8192,"text,image",0,null,0.1,0.4,0.025,0],["gemini-2.0-flash-lite",1048576,8192,"text,image",0,null,0.075,0.3,0,0],["gemini-2.5-computer-use-preview-10-2025",131072,65536,"text,image",1,null,1.25,10,0,0],["gemini-2.5-flash",1048576,65536,"text,image",1,null,0.3,2.5,0.03,0],["gemini-2.5-flash-lite",1048576,65536,"text,image",1,null,0.1,0.4,0.01,0],["gemini-2.5-flash-lite-preview-06-17",1048576,65536,"text,image",1,null,0.1,0.4,0.025,0],["gemini-2.5-flash-lite-preview-09-2025",1048576,65536,"text,image",1,null,0.1,0.4,0.025,0],["gemini-2.5-flash-preview-04-17",1048576,65536,"text,image",1,null,0.15,0.6,0.0375,0],["gemini-2.5-flash-preview-05-20",1048576,65536,"text,image",1,null,0.15,0.6,0.0375,0],["gemini-2.5-flash-preview-09-2025",1048576,65536,"text,image",1,null,0.3,2.5,0.075,0],["gemini-2.5-pro",1048576,65536,"text,image",1,null,1.25,10,0.125,0],["gemini-2.5-pro-preview-05-06",1048576,65536,"text,image",1,null,1.25,10,0.31,0],["gemini-2.5-pro-preview-06-05",1048576,65536,"text,image",1,null,1.25,10,0.31,0],["gemini-3-flash-preview",1048576,65536,"text,image",1,null,0.5,3,0.05,0],["gemini-3-pro-preview",1048576,65536,"text,image",1,null,2,12,0.2,0],["gemini-3.1-flash-lite",1048576,65536,"text,image",1,null,0.25,1.5,0.025,0],["gemini-3.1-flash-lite-image",65536,65536,"text,image",1,null,0.25,30,0,0],["gemini-3.1-flash-lite-preview",1048576,65536,"text,image",1,null,0.25,1.5,0.025,0],["gemini-3.1-flash-live-preview",131072,65536,"text,image",1,null,0.75,4.5,0,0],["gemini-3.1-pro-preview",1048576,65536,"text,image",1,null,2,12,0.2,0],["gemini-3.1-pro-preview-customtools",1048576,65536,"text,image",1,null,2,12,0.2,0],["gemini-3.5-flash",1048576,65536,"text,image",1,null,1.5,9,0.15,0],["gemini-3.5-flash-lite",1048576,65536,"text,image",1,null,0.3,2.5,0.03,0],["gemini-3.6-flash",1048576,65536,"text,image",1,null,1.5,7.5,0.15,0],["gemini-flash-latest",1048576,65536,"text,image",1,null,1.5,9,0.15,0],["gemini-flash-lite-latest",1048576,65536,"text,image",1,null,0.25,1.5,0.025,0],["gemini-live-2.5-flash",128000,8000,"text,image",1,null,0.5,2,0,0],["gemini-live-2.5-flash-preview-native-audio",131072,65536,"text",1,null,0.5,2,0,0],["gemini-robotics-er-1.6-preview",131072,65536,"text,image",1,null,1,5,0,0],["gemma-3-27b-it",131072,8192,"text,image",0,null,0,0,0,0],["gemma-4-26b",256000,8192,"text,image",1,null,0,0,0,0],["gemma-4-26b-a4b-it",262144,32768,"text,image",1,null,0,0,0,0],["gemma-4-26b-it",256000,8192,"text,image",1,null,0,0,0,0],["gemma-4-31b",256000,8192,"text,image",1,null,0,0,0,0],["gemma-4-31b-it",262144,32768,"text,image",1,null,0,0,0,0],["gemma-4-E2B-it",131072,8192,"text,image",1,null,0,0,0,0],["gemma-4-E4B-it",131072,8192,"text,image",1,null,0,0,0,0]], "minimax": [["MiniMax-M2",196608,128000,"text",1,null,0.3,1.2,0,0],["MiniMax-M2.1",204800,131072,"text",1,null,0.3,1.2,0,0],["MiniMax-M2.5",204800,131072,"text",1,null,0.3,1.2,0.03,0.375],["MiniMax-M2.5-highspeed",204800,131072,"text",1,null,0.6,2.4,0.06,0.375],["MiniMax-M2.5-lightning",204800,32000,"text",1,null,0.3,2.4,0,0],["MiniMax-M2.7",204800,131072,"text",1,null,0.3,1.2,0.06,0.375],["MiniMax-M2.7-highspeed",204800,131072,"text",1,null,0.6,2.4,0.06,0.375],["minimax-m3",512000,128000,"text,image",1,null,0.6,2.4,0.12,0],["MiniMax-M3",1000000,128000,"text,image,video",1,null,0.3,1.2,0.06,0]], "mistral": [["codestral-latest",256000,4096,"text",0,null,0.3,0.9,0,0],["devstral-2512",262144,262144,"text",0,null,0.4,2,0,0],["devstral-latest",262144,262144,"text",0,null,0.4,2,0,0],["devstral-medium-2507",128000,128000,"text",0,null,0.4,2,0,0],["devstral-medium-latest",262144,262144,"text",0,null,0.4,2,0,0],["devstral-small-2505",128000,128000,"text",0,null,0.1,0.3,0,0],["devstral-small-2507",128000,128000,"text",0,null,0.1,0.3,0,0],["labs-devstral-small-2512",256000,256000,"text,image",0,null,0,0,0,0],["magistral-medium-latest",128000,16384,"text",1,null,2,5,0,0],["magistral-small",128000,128000,"text",1,null,0.5,1.5,0,0],["ministral-3b-latest",128000,128000,"text",0,null,0.04,0.04,0,0],["ministral-8b-latest",128000,128000,"text",0,null,0.1,0.1,0,0],["mistral-large-2411",131072,16384,"text",0,null,2,6,0,0],["mistral-large-2512",262144,262144,"text,image",0,null,0.5,1.5,0,0],["mistral-large-latest",262144,262144,"text,image",0,null,0.5,1.5,0,0],["mistral-medium-2505",131072,131072,"text,image",0,null,0.4,2,0,0],["mistral-medium-2508",262144,262144,"text,image",0,null,0.4,2,0,0],["mistral-medium-2604",262144,262144,"text,image",1,null,1.5,7.5,0,0],["mistral-medium-latest",262144,262144,"text,image",1,null,1.5,7.5,0,0],["mistral-nemo",128000,128000,"text",0,null,0.15,0.15,0,0],["mistral-small-2506",128000,16384,"text,image",0,null,0.1,0.3,0,0],["mistral-small-2603",256000,256000,"text,image",1,null,0.15,0.6,0,0],["mistral-small-latest",256000,256000,"text,image",1,null,0.15,0.6,0,0],["open-mistral-7b",8000,8000,"text",0,null,0.25,0.25,0,0],["open-mistral-nemo",128000,128000,"text",0,null,0.15,0.15,0,0],["open-mixtral-8x22b",64000,64000,"text",0,null,2,6,0,0],["open-mixtral-8x7b",32000,32000,"text",0,null,0.7,0.7,0,0],["pixtral-12b",128000,128000,"text,image",0,null,0.15,0.15,0,0],["pixtral-large-latest",128000,128000,"text,image",0,null,2,6,0,0]], @@ -54,7 +55,7 @@ const DATA: Record = { }; export function resolveMetadataProvider(provider: string): string | undefined { - return PROVIDER_ALIASES[provider]; + return PROVIDER_ALIASES[provider] ?? PROVIDER_ALIASES[provider.toLowerCase()]; } export function getModelMetadata(provider: string, modelId: string): ModelMetadata | undefined { diff --git a/src/providers/registry.ts b/src/providers/registry.ts index b9d6d50cb..656816253 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1300,14 +1300,15 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ authKind: "key", dashboardUrl: "https://platform.deepseek.com/api_keys", // Route DeepSeek's own catalog bundle so routed rebuilds restore the official - // context window from jawcode metadata instead of falling back to the 128k - // strict-fields default (jawcode models.json, verified 2026-08-08). + // context window from the vendored model-metadata bundle instead of falling + // back to the 128k strict-fields default (scripts/model-metadata.source.json, + // verified 2026-08-08). jawcodeBundle: "deepseek", // deepseek-chat/deepseek-reasoner were deprecated upstream on 2026-07-24 15:59 UTC; // official identifiers are now deepseek-v4-flash / deepseek-v4-pro. They stay in // the list only as compatibility aliases so existing saved configs and requests // keep validating and routing (they previously mapped to v4-flash; devlog - // _plan/260710_provider_hardening/002_research_cn.md). The current offerings are + // _fin/260710_provider_hardening/002_research_cn.md). The current offerings are // the V4 ids — defaultModel and the model-specific wiring above use them. models: ["deepseek-chat", "deepseek-reasoner", ...DEEPSEEK_THINKING_MODELS], defaultModel: "deepseek-v4-flash", From 7f536489186e7b811d7cd0ea97097d72b16e8d2c Mon Sep 17 00:00:00 2001 From: xinweigao Date: Sat, 8 Aug 2026 18:49:59 +0800 Subject: [PATCH 04/77] fix(catalog): restore DeepSeek V4 rows when /v1/models returns an empty list Add deepseek to the catalog augmentation allowlist so metadata-sourced V4 rows are appended when live discovery returns nothing, and cover the empty-discovery path with a regression test asserting the official 1,048,576 context window, auto-compact limit, and text-only input. --- src/codex/catalog/parsing.ts | 2 +- tests/codex-catalog.test.ts | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/codex/catalog/parsing.ts b/src/codex/catalog/parsing.ts index a4cc9e073..fa0028759 100644 --- a/src/codex/catalog/parsing.ts +++ b/src/codex/catalog/parsing.ts @@ -126,7 +126,7 @@ export type RawEntry = Record; export type RawCatalog = { models?: RawEntry[]; [k: string]: unknown }; -export const JAWCODE_CATALOG_AUGMENT_PROVIDERS = new Set(["opencode-go"]); +export const JAWCODE_CATALOG_AUGMENT_PROVIDERS = new Set(["opencode-go", "deepseek"]); export const ROUTED_MODEL_COMPATIBILITY_EXCLUSIONS = new Set([ // Issue #82: Zen Go /models advertises HY3, but Console Go rejects it as outside the lite list. diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index 12cba1b04..a46835463 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -2846,6 +2846,27 @@ describe("Codex catalog routed normalization", () => { expect(getModelMetadata("deepseek", "deepseek-v4-flash")?.contextWindow).toBe(1_048_576); }); + test("DeepSeek catalog sync appends V4 rows missing from /v1/models", () => { + const models = augmentRoutedModelsWithMetadata([], ["deepseek"]); + const slugs = new Set(models.map(m => `${m.provider}/${m.id}`)); + + expect(slugs.has("deepseek/deepseek-v4-flash")).toBe(true); + expect(slugs.has("deepseek/deepseek-v4-pro")).toBe(true); + for (const model of models) { + expect(model.contextWindow).toBe(1_048_576); + expect(model.inputModalities).toEqual(["text"]); + } + + const entries = buildCatalogEntries(nativeTemplate(), [], models); + for (const id of ["deepseek-v4-flash", "deepseek-v4-pro"]) { + const routed = entries.find(e => e.slug === `deepseek/${id}`); + expect(routed?.context_window).toBe(1_048_576); + expect(routed?.max_context_window).toBe(1_048_576); + expect(routed?.auto_compact_token_limit).toBe(943_718); // floor(1048576 * 0.9) + expect(routed?.input_modalities).toEqual(["text"]); + } + }); + test("provider context-cap applies before jawcode catalog metadata reaches Codex", () => { const entries = buildCatalogEntries(nativeTemplate(), [], [ { provider: "opencode-go", id: "deepseek-v4-pro", contextCap: 350_000, contextCapped: false }, From b8bfa2c265d396168fcae5b462eadcf816c3a618 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:26:49 +0200 Subject: [PATCH 05/77] test(routing): lock policy evidence parity across inbound protocols --- tests/routing-policy-surface-parity.test.ts | 103 ++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/routing-policy-surface-parity.test.ts diff --git a/tests/routing-policy-surface-parity.test.ts b/tests/routing-policy-surface-parity.test.ts new file mode 100644 index 000000000..a8bbe1e89 --- /dev/null +++ b/tests/routing-policy-surface-parity.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, test } from "bun:test"; + +import { chatCompletionsToResponsesBody } from "../src/chat/inbound"; +import { anthropicToResponsesTranslation } from "../src/claude/inbound"; +import { evidenceFromBody } from "../src/routing/request-evidence"; + +const MODEL = "policy/daily"; +const EXPECTED_RICH_EVIDENCE = { + toolsRequired: true, + imageInputRequired: true, +}; + +describe("routing policy request evidence parity", () => { + test("tools and image input produce the same evidence across Responses, Chat Completions, and Claude Messages", () => { + const responsesBody = { + model: MODEL, + input: [{ + type: "message", + role: "user", + content: [ + { type: "input_text", text: "inspect this" }, + { type: "input_image", image_url: "data:image/png;base64,AA==" }, + ], + }], + tools: [{ + type: "function", + name: "inspect", + parameters: { type: "object", properties: {} }, + }], + }; + + const chatBody = chatCompletionsToResponsesBody({ + model: MODEL, + messages: [{ + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AA==" } }, + ], + }], + tools: [{ + type: "function", + function: { + name: "inspect", + parameters: { type: "object", properties: {} }, + }, + }], + }); + + const claudeBody = anthropicToResponsesTranslation({ + model: MODEL, + max_tokens: 128, + messages: [{ + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "AA==", + }, + }, + ], + }], + tools: [{ + name: "inspect", + input_schema: { type: "object", properties: {} }, + }], + }).body; + + expect(evidenceFromBody(responsesBody)).toEqual(EXPECTED_RICH_EVIDENCE); + expect(evidenceFromBody(chatBody)).toEqual(EXPECTED_RICH_EVIDENCE); + expect(evidenceFromBody(claudeBody)).toEqual(EXPECTED_RICH_EVIDENCE); + }); + + test("plain text without tools produces no hard routing evidence on every surface", () => { + const responsesBody = { + model: MODEL, + input: [{ + type: "message", + role: "user", + content: [{ type: "input_text", text: "hello" }], + }], + }; + + const chatBody = chatCompletionsToResponsesBody({ + model: MODEL, + messages: [{ role: "user", content: "hello" }], + }); + + const claudeBody = anthropicToResponsesTranslation({ + model: MODEL, + max_tokens: 128, + messages: [{ role: "user", content: "hello" }], + }).body; + + expect(evidenceFromBody(responsesBody)).toEqual({}); + expect(evidenceFromBody(chatBody)).toEqual({}); + expect(evidenceFromBody(claudeBody)).toEqual({}); + }); +}); From 82afb71db7064a450f2a1693a73418a3f55deded Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:30:08 +0200 Subject: [PATCH 06/77] test(routing): define policy candidate fallback behavior --- tests/routing-policy-fallback.test.ts | 155 ++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 tests/routing-policy-fallback.test.ts diff --git a/tests/routing-policy-fallback.test.ts b/tests/routing-policy-fallback.test.ts new file mode 100644 index 000000000..11e9360d9 --- /dev/null +++ b/tests/routing-policy-fallback.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from "bun:test"; + +import type { OcxConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; +import type { RouteDecisionTraceV1 } from "../src/routing/trace"; +import { + handleResponsesWithPolicyFallback, + rankPolicyFallbackCandidates, +} from "../src/server/responses/policy-fallback"; + +function policyTrace(): RouteDecisionTraceV1 { + return { + version: 1, + decisionId: "decision-1", + createdAt: 1, + requestedModel: "policy/daily", + routeKind: "policy", + profile: { id: "daily", revision: "rev-1" }, + requirements: [], + candidates: [ + { + provider: "provider-a", + model: "model-a", + eligible: true, + exclusions: [], + score: { total: 0.90, components: {} }, + }, + { + provider: "provider-b", + model: "model-b", + eligible: true, + exclusions: [], + score: { total: 0.80, components: {} }, + }, + { + provider: "provider-c", + model: "model-c", + eligible: true, + exclusions: [], + score: { total: 0.80, components: {} }, + }, + { + provider: "provider-d", + model: "model-d", + eligible: false, + exclusions: [{ code: "tools" }], + score: { total: 1, components: {} }, + }, + ], + selected: { + candidateIndex: 0, + provider: "provider-a", + model: "model-a", + reason: "highest-score", + }, + }; +} + +function request(): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "policy/daily", input: "hello", stream: false }), + }); +} + +describe("policy candidate fallback", () => { + test("ranks only eligible untried candidates by score and stable original order", () => { + const trace = policyTrace(); + const ranked = rankPolicyFallbackCandidates( + trace, + new Set(["provider-a\u0000model-a"]), + ); + + expect(ranked.map(candidate => `${candidate.provider}/${candidate.model}`)).toEqual([ + "provider-b/model-b", + "provider-c/model-c", + ]); + }); + + test("retries the next policy candidate after a retryable pre-stream failure", async () => { + const trace = policyTrace(); + const logCtx = { + requestedModel: "policy/daily", + routeDecision: trace, + attempts: [], + } as unknown as RequestLogContext; + const seenModels: string[] = []; + + const response = await handleResponsesWithPolicyFallback( + request(), + {} as OcxConfig, + logCtx, + {}, + { + runCore: async (req, _config, childLog) => { + const body = await req.json() as { model: string }; + seenModels.push(body.model); + if (seenModels.length === 1) { + childLog.requestedModel = "policy/daily"; + childLog.routeDecision = trace; + return new Response( + JSON.stringify({ error: { message: "rate limited", type: "rate_limit_error" } }), + { status: 429, headers: { "content-type": "application/json" } }, + ); + } + childLog.requestedModel = body.model; + childLog.routeDecision = { + ...trace, + requestedModel: body.model, + routeKind: "explicit-provider", + profile: undefined, + }; + return new Response(JSON.stringify({ status: "completed" }), { status: 200 }); + }, + }, + ); + + expect(response.status).toBe(200); + expect(seenModels).toEqual(["policy/daily", "provider-b/model-b"]); + expect(logCtx.requestedModel).toBe("policy/daily"); + expect(logCtx.routeDecision).toBe(trace); + }); + + test("does not switch candidates for terminal client/input failures", async () => { + const trace = policyTrace(); + const logCtx = { + requestedModel: "policy/daily", + routeDecision: trace, + attempts: [], + } as unknown as RequestLogContext; + let calls = 0; + + const response = await handleResponsesWithPolicyFallback( + request(), + {} as OcxConfig, + logCtx, + {}, + { + runCore: async (_req, _config, childLog) => { + calls += 1; + childLog.requestedModel = "policy/daily"; + childLog.routeDecision = trace; + return new Response( + JSON.stringify({ error: { message: "invalid request", type: "invalid_request_error" } }), + { status: 400, headers: { "content-type": "application/json" } }, + ); + }, + }, + ); + + expect(response.status).toBe(400); + expect(calls).toBe(1); + }); +}); From cd7ea8a8822f81d352e1e2dfdb726ba1d7afb0c3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:41:17 +0200 Subject: [PATCH 07/77] feat(routing): add policy candidate fallback wrapper --- src/server/responses/policy-fallback.ts | 156 ++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/server/responses/policy-fallback.ts diff --git a/src/server/responses/policy-fallback.ts b/src/server/responses/policy-fallback.ts new file mode 100644 index 000000000..5a701c6f4 --- /dev/null +++ b/src/server/responses/policy-fallback.ts @@ -0,0 +1,156 @@ +import { comboFailureDecision } from "../../combos/failover"; +import { readBoundedResponseBody } from "../../lib/bounded-body"; +import { readJsonRequestBody } from "../request-decompress"; +import type { RequestLogContext } from "../request-log"; +import type { OcxConfig } from "../../types"; +import type { RouteCandidateTrace, RouteDecisionTraceV1 } from "../../routing/trace"; +import { handleResponses as handleResponsesCore } from "./core"; + +type CoreHandler = typeof handleResponsesCore; +type CoreOptions = Parameters[3]; + +export interface PolicyFallbackDeps { + runCore?: CoreHandler; +} + +function candidateKey(candidate: Pick): string { + return `${candidate.provider}\u0000${candidate.model}`; +} + +/** + * Rank the remaining candidates from the ORIGINAL policy trace. The initial + * decision stays immutable; fallback execution belongs in attempts[], not in a + * rewritten decision trace. + */ +export function rankPolicyFallbackCandidates( + trace: RouteDecisionTraceV1, + tried: ReadonlySet, +): RouteCandidateTrace[] { + return trace.candidates + .map((candidate, index) => ({ candidate, index })) + .filter(({ candidate }) => + candidate.eligible + && candidate.exclusions.length === 0 + && !tried.has(candidateKey(candidate))) + .sort((left, right) => { + const scoreDelta = (right.candidate.score?.total ?? Number.NEGATIVE_INFINITY) + - (left.candidate.score?.total ?? Number.NEGATIVE_INFINITY); + return scoreDelta || left.index - right.index; + }) + .map(({ candidate }) => candidate); +} + +function requestWithCandidate( + req: Request, + rawBody: Record, + candidate: Pick, +): Request { + const headers = new Headers(req.headers); + // The retry body is re-serialized JSON. Carrying the original compression or + // byte length would make the child request malformed. + headers.delete("content-encoding"); + headers.delete("content-length"); + headers.set("content-type", "application/json"); + return new Request(req.url, { + method: req.method, + headers, + body: JSON.stringify({ + ...rawBody, + model: `${candidate.provider}/${candidate.model}`, + }), + signal: req.signal, + }); +} + +function errorCodeFromText(text: string): string | undefined { + if (!text) return undefined; + try { + const payload = JSON.parse(text) as { + error?: { code?: unknown; type?: unknown }; + code?: unknown; + }; + const candidate = payload.error?.code ?? payload.error?.type ?? payload.code; + return typeof candidate === "string" ? candidate : undefined; + } catch { + return undefined; + } +} + +async function shouldHopPolicyCandidate(response: Response, signal?: AbortSignal): Promise { + if (response.status < 400) return false; + try { + const inspected = await readBoundedResponseBody(response.clone(), { signal }); + const text = inspected.displaySafe ? inspected.text : ""; + return comboFailureDecision(response.status, text, { + code: errorCodeFromText(text), + }) === "hop"; + } catch { + // If the error body cannot be inspected safely, do not invent a retry. + return false; + } +} + +function isPolicyDecision(trace: RouteDecisionTraceV1 | undefined): trace is RouteDecisionTraceV1 { + return trace?.routeKind === "policy" && !!trace.profile; +} + +/** + * Run a Responses request and, only for an explicitly selected policy profile, + * hop to the next eligible policy candidate after a retryable pre-success + * failure. The initial policy trace remains the canonical selection evidence; + * physical retries continue to accumulate in the existing request attempts. + */ +export async function handleResponsesWithPolicyFallback( + req: Request, + config: OcxConfig, + logCtx: RequestLogContext, + options: CoreOptions = {}, + deps: PolicyFallbackDeps = {}, +): Promise { + const runCore = deps.runCore ?? handleResponsesCore; + + // Capture a replayable, decompressed body before the core consumes the + // request. If decoding fails, defer entirely to the canonical core path so + // its existing error semantics remain unchanged. + let rawBody: Record | null = null; + try { + const parsed = await readJsonRequestBody(req.clone()); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + rawBody = parsed as Record; + } + } catch { + // Core owns the client-facing parse/decompression error. + } + + let response = await runCore(req, config, logCtx, options); + const initialTrace = logCtx.routeDecision; + const initialRequestedModel = logCtx.requestedModel; + if (!rawBody || !isPolicyDecision(initialTrace)) return response; + + const tried = new Set([ + candidateKey({ provider: initialTrace.selected.provider, model: initialTrace.selected.model }), + ]); + + while (await shouldHopPolicyCandidate(response, req.signal)) { + if (req.signal.aborted) return response; + const next = rankPolicyFallbackCandidates(initialTrace, tried)[0]; + if (!next) return response; + tried.add(candidateKey(next)); + + const retryRequest = requestWithCandidate(req, rawBody, next); + try { + response = await runCore(retryRequest, config, logCtx, options); + } finally { + // A fallback child routes explicitly and therefore produces its own + // explicit-provider trace. Keep the original policy decision as the + // request-level WHY while retaining the child's physical model/provider + // and attempts on the mutable log context. + logCtx.requestedModel = initialRequestedModel; + logCtx.routeDecision = initialTrace; + } + } + + return response; +} + +export const handleResponses = handleResponsesWithPolicyFallback; From 457c33675715b0a5f5ccee314a5ef94f42e4b62a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:41:35 +0200 Subject: [PATCH 08/77] feat(routing): route Responses through policy fallback --- src/server/responses.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/responses.ts b/src/server/responses.ts index 8c33c5bca..172baccbc 100644 --- a/src/server/responses.ts +++ b/src/server/responses.ts @@ -5,5 +5,6 @@ export type { MultiAgentGuidanceOptions, MultiAgentGuidanceDeps } from "./respon export { hasUnreadableEncryptedAgentTask, sanitizeEncryptedContentInPlace } from "./responses/encrypted-payload"; export { COMPACT_RESPONSE_MAX_BYTES, bufferCompactResponse, handleResponsesCompact } from "./responses/compact"; export { disableResponsesRequestTimeout, safeHostLabel, fetchWithHeaderTimeout } from "./responses/fetch-helpers"; -export { sidecarOutcomeRecorder, isShadowSourceModel, codexLogAccountId, usesCodexForwardPoolAuth, codexForwardTerminalOutcomeRecorder, decodeRequestErrorResponse, buildComboChildHeaders, handleResponses, linkAbortSignal } from "./responses/core"; -export { adapterNeedsForcedContinuation } from "./responses/core"; +export { sidecarOutcomeRecorder, isShadowSourceModel, codexLogAccountId, usesCodexForwardPoolAuth, codexForwardTerminalOutcomeRecorder, decodeRequestErrorResponse, buildComboChildHeaders, linkAbortSignal } from "./responses/core"; +export { handleResponses, handleResponsesWithPolicyFallback, rankPolicyFallbackCandidates } from "./responses/policy-fallback"; +export { adapterNeedsForcedContinuation } from "./responses/core"; \ No newline at end of file From fb7a498ecc588d013a1f7bf83d107bb510f19224 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:44:08 +0200 Subject: [PATCH 09/77] test(routing): define pool-aware policy quota evidence --- tests/routing-policy-pool-quota.test.ts | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/routing-policy-pool-quota.test.ts diff --git a/tests/routing-policy-pool-quota.test.ts b/tests/routing-policy-pool-quota.test.ts new file mode 100644 index 000000000..f1e7da778 --- /dev/null +++ b/tests/routing-policy-pool-quota.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { clearAccountQuota, setAccountQuotaFromParsed } from "../src/codex/quota"; +import { codexPoolQuotaEvidence } from "../src/routing/quota"; + +afterEach(() => clearAccountQuota()); + +describe("Codex pool quota evidence for routing policies", () => { + test("uses the best known usable headroom instead of only one active account", () => { + setAccountQuotaFromParsed("low", { weeklyPercent: 95 }); + setAccountQuotaFromParsed("healthy", { weeklyPercent: 20 }); + + expect(codexPoolQuotaEvidence([ + { accountId: "low", plan: "plus" }, + { accountId: "healthy", plan: "plus" }, + ])).toMatchObject({ + known: true, + exhausted: false, + headroom: 0.8, + source: "codex-pool", + }); + }); + + test("reports exhausted only when every pool account is known exhausted", () => { + setAccountQuotaFromParsed("a", { weeklyPercent: 100, weeklyResetAt: Date.now() + 60_000 }); + setAccountQuotaFromParsed("b", { weeklyPercent: 100, weeklyResetAt: Date.now() + 120_000 }); + + const evidence = codexPoolQuotaEvidence([ + { accountId: "a", plan: "plus" }, + { accountId: "b", plan: "plus" }, + ]); + + expect(evidence.known).toBe(true); + expect(evidence.exhausted).toBe(true); + expect(evidence.headroom).toBe(0); + expect(evidence.resetAtMs).toBeDefined(); + }); + + test("does not call a partially unknown pool exhausted", () => { + setAccountQuotaFromParsed("known-exhausted", { weeklyPercent: 100 }); + + expect(codexPoolQuotaEvidence([ + { accountId: "known-exhausted", plan: "plus" }, + { accountId: "unknown", plan: "plus" }, + ])).toEqual({ known: false }); + }); +}); From ba00ac79e8de788378aa33ee283f79d372d43a00 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:46:06 +0200 Subject: [PATCH 10/77] feat(routing): aggregate Codex pool quota evidence --- src/routing/quota.ts | 120 +++++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 27 deletions(-) diff --git a/src/routing/quota.ts b/src/routing/quota.ts index 06cc8eecc..3d0bd6bd7 100644 --- a/src/routing/quota.ts +++ b/src/routing/quota.ts @@ -12,7 +12,12 @@ * carries an account reference (dry-run/evaluate), never invented. */ -import { codexQuotaWindowForPlan, getAccountQuota, isCodexQuotaExhausted } from "../codex/quota"; +import { + codexQuotaWindowForPlan, + getAccountQuota, + isCodexQuotaExhausted, + listAccountQuotas, +} from "../codex/quota"; import { getCachedProviderAccountQuota } from "../providers/quota"; import type { RouteQuotaEvidence } from "./trace"; @@ -27,39 +32,100 @@ export interface QuotaEvidenceInput { codexAccountPlan?: string; } +export interface CodexPoolQuotaAccount { + accountId: string; + plan?: string; +} + +function codexAccountQuotaEvidence(accountId: string, plan?: string): RouteQuotaEvidence { + const quota = getAccountQuota(accountId); + if (!quota) return { known: false }; + + // Go/Free accounts report a 30-day window only; weekly windows gate + // everything else. `codexQuotaWindowForPlan` is the single shared rule + // (parser, exhaustion, recovery), so select the plan-specific bars here too. + const monthly = codexQuotaWindowForPlan(plan) === "monthly"; + const percents = [ + ...(monthly ? [] : [quota.weeklyPercent]), + quota.monthlyPercent, + ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; + const resets = [ + ...(monthly ? [] : [quota.weeklyResetAt]), + quota.monthlyResetAt, + ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)) + .filter(value => value > Date.now()); + return { + known: true, + ...(maxPercent !== undefined + ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } + : {}), + exhausted: isCodexQuotaExhausted(quota, plan), + ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), + source: "codex-pool", + }; +} + +/** + * Provider-level quota evidence for a Codex account pool. A policy profile + * chooses provider/model, while the existing pool remains authoritative for + * the physical account. Therefore the provider is usable when ANY known pool + * account has headroom. Unknown accounts prevent a known-exhausted verdict: + * unknown capacity is not zero capacity. + */ +export function codexPoolQuotaEvidence(accounts: readonly CodexPoolQuotaAccount[]): RouteQuotaEvidence { + if (accounts.length === 0) return { known: false }; + const evidence = accounts.map(account => codexAccountQuotaEvidence(account.accountId, account.plan)); + const known = evidence.filter(item => item.known); + if (known.length === 0) return { known: false }; + + const usable = known.filter(item => item.exhausted !== true); + if (usable.length > 0) { + const headrooms = usable + .map(item => item.headroom) + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + return { + known: true, + exhausted: false, + ...(headrooms.length > 0 ? { headroom: Math.max(...headrooms) } : {}), + source: "codex-pool", + }; + } + + // At least one account has no quota evidence. The pool may still be usable, + // so fail open as unknown rather than excluding the provider as exhausted. + if (known.length < evidence.length) return { known: false }; + + const resets = known + .map(item => item.resetAtMs) + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + return { + known: true, + exhausted: true, + headroom: 0, + ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), + source: "codex-pool", + }; +} + /** * Assemble quota evidence from canonical local caches only (no network). * Unknown dimensions stay unknown - never zero. */ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuotaEvidence { if (input.provider === "openai" && input.codexAccountId) { - const quota = getAccountQuota(input.codexAccountId); - if (quota) { - // Go/Free accounts report a 30-day window only; weekly windows gate - // everything else. `codexQuotaWindowForPlan` is the single shared rule - // (parser, exhaustion, recovery), so select the plan-specific bars here - // too instead of always combining weekly + monthly. - const monthly = codexQuotaWindowForPlan(input.codexAccountPlan) === "monthly"; - const percents = [ - ...(monthly ? [] : [quota.weeklyPercent]), - quota.monthlyPercent, - ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)); - const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; - const resets = [ - ...(monthly ? [] : [quota.weeklyResetAt]), - quota.monthlyResetAt, - ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)) - .filter(value => value > Date.now()); - return { - known: true, - ...(maxPercent !== undefined - ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } - : {}), - exhausted: isCodexQuotaExhausted(quota, input.codexAccountPlan), - ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), - source: "codex-pool", - }; + // The live routing/profile assembly path includes the selected account's + // plan. At that boundary the candidate represents the whole Codex pool, + // not that one active account, so aggregate the reconciled quota cache. + // Caller-supplied dry-run account evidence omits the plan and remains exact. + if (input.codexAccountPlan !== undefined) { + const pool = [...listAccountQuotas()].map(([accountId]) => ({ + accountId, + ...(accountId === input.codexAccountId ? { plan: input.codexAccountPlan } : {}), + })); + if (pool.length > 1) return codexPoolQuotaEvidence(pool); } + return codexAccountQuotaEvidence(input.codexAccountId, input.codexAccountPlan); } if (input.provider === "anthropic" && input.accountRef) { From 62ed87acf20fd6fc8506425868024a35619796b9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:47:25 +0200 Subject: [PATCH 11/77] test(routing): cover live policy pool aggregation path --- tests/routing-policy-pool-quota.test.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/routing-policy-pool-quota.test.ts b/tests/routing-policy-pool-quota.test.ts index f1e7da778..24dea36b0 100644 --- a/tests/routing-policy-pool-quota.test.ts +++ b/tests/routing-policy-pool-quota.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { clearAccountQuota, setAccountQuotaFromParsed } from "../src/codex/quota"; -import { codexPoolQuotaEvidence } from "../src/routing/quota"; +import { codexPoolQuotaEvidence, quotaEvidenceForCandidate } from "../src/routing/quota"; afterEach(() => clearAccountQuota()); @@ -21,6 +21,23 @@ describe("Codex pool quota evidence for routing policies", () => { }); }); + test("the live policy evidence path aggregates the reconciled pool", () => { + setAccountQuotaFromParsed("active", { weeklyPercent: 96 }); + setAccountQuotaFromParsed("alternate", { weeklyPercent: 25 }); + + expect(quotaEvidenceForCandidate({ + provider: "openai", + model: "gpt-5.6", + codexAccountId: "active", + codexAccountPlan: "plus", + })).toMatchObject({ + known: true, + exhausted: false, + headroom: 0.75, + source: "codex-pool", + }); + }); + test("reports exhausted only when every pool account is known exhausted", () => { setAccountQuotaFromParsed("a", { weeklyPercent: 100, weeklyResetAt: Date.now() + 60_000 }); setAccountQuotaFromParsed("b", { weeklyPercent: 100, weeklyResetAt: Date.now() + 120_000 }); @@ -44,4 +61,4 @@ describe("Codex pool quota evidence for routing policies", () => { { accountId: "unknown", plan: "plus" }, ])).toEqual({ known: false }); }); -}); +}); \ No newline at end of file From 130165818fbb3c8c94df9821dc16620ada5bb85d Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:53:29 +0000 Subject: [PATCH 12/77] fix: apply CodeRabbit auto-fixes Fixed 1 file(s) based on 1 unresolved review comment. Co-authored-by: CodeRabbit --- tests/routing-policy-surface-parity.test.ts | 269 +++++++++++++++++++- 1 file changed, 267 insertions(+), 2 deletions(-) diff --git a/tests/routing-policy-surface-parity.test.ts b/tests/routing-policy-surface-parity.test.ts index a8bbe1e89..9b466b4dc 100644 --- a/tests/routing-policy-surface-parity.test.ts +++ b/tests/routing-policy-surface-parity.test.ts @@ -1,8 +1,11 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, mock, test } from "bun:test"; import { chatCompletionsToResponsesBody } from "../src/chat/inbound"; import { anthropicToResponsesTranslation } from "../src/claude/inbound"; import { evidenceFromBody } from "../src/routing/request-evidence"; +import type { ProviderAdapter } from "../src/adapters/base"; +import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; const MODEL = "policy/daily"; const EXPECTED_RICH_EVIDENCE = { @@ -10,7 +13,7 @@ const EXPECTED_RICH_EVIDENCE = { imageInputRequired: true, }; -describe("routing policy request evidence parity", () => { +describe("routing policy request evidence parity (translator-level coverage)", () => { test("tools and image input produce the same evidence across Responses, Chat Completions, and Claude Messages", () => { const responsesBody = { model: MODEL, @@ -101,3 +104,265 @@ describe("routing policy request evidence parity", () => { expect(evidenceFromBody(claudeBody)).toEqual({}); }); }); + +// ---- Handler-level parity tests (via dev handler entry points) ---- + +const actualResolver = await import("../src/server/adapter-resolve"); +let adapterFactory: ((provider: OcxProviderConfig) => ProviderAdapter) | undefined; + +mock.module("../src/server/adapter-resolve", () => ({ + ...actualResolver, + resolveAdapter(provider: OcxProviderConfig, cacheRetention?: "none" | "short" | "long") { + return adapterFactory?.(provider) ?? actualResolver.resolveAdapter(provider, cacheRetention); + }, +})); + +const { handleResponses } = await import("../src/server/responses"); +const { handleChatCompletions } = await import("../src/server/chat-completions"); +const { handleClaudeMessages } = await import("../src/server/claude-messages"); + +afterEach(() => { + adapterFactory = undefined; +}); + +function testConfig(): OcxConfig { + return { + port: 0, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + models: ["m1"], + modelContextWindows: { m1: 200_000 }, + modelInputModalities: { m1: ["text", "image"] }, + parallelToolCalls: true, + }, + }, + routingProfiles: { + daily: { candidates: [{ provider: "a", model: "m1" }] }, + }, + } as OcxConfig; +} + +function minimalSuccessAdapter(provider: OcxProviderConfig): ProviderAdapter { + return { + name: "test-run-turn", + buildRequest: () => ({ url: provider.baseUrl, method: "POST", headers: {}, body: "" }), + async *parseStream(): AsyncGenerator { + yield { type: "error", message: "test runTurn adapter does not use parseStream" }; + }, + async runTurn(_parsed, _incoming, emit) { + emit({ type: "text_delta", text: "ok" }); + emit({ type: "done" }); + }, + }; +} + +describe("routing policy request evidence parity (via dev handlers)", () => { + test("rich evidence (tools + image) produces identical route decision across all three surfaces", async () => { + adapterFactory = minimalSuccessAdapter; + const config = testConfig(); + + // Responses: native input[] shape + const responsesLogCtx: RequestLogContext = { model: "", provider: "" }; + const responsesReq = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: MODEL, + stream: false, + input: [{ + type: "message", + role: "user", + content: [ + { type: "input_text", text: "inspect this" }, + { type: "input_image", image_url: "data:image/png;base64,AA==" }, + ], + }], + tools: [{ + type: "function", + name: "inspect", + parameters: { type: "object", properties: {} }, + }], + }), + }); + const responsesResponse = await handleResponses(responsesReq, config, responsesLogCtx); + await responsesResponse.text(); + + // Chat Completions: OpenAI messages[] shape + const chatLogCtx: RequestLogContext = { model: "", provider: "" }; + const chatReq = new Request("http://localhost/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: MODEL, + stream: false, + messages: [{ + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AA==" } }, + ], + }], + tools: [{ + type: "function", + function: { + name: "inspect", + parameters: { type: "object", properties: {} }, + }, + }], + }), + }); + const chatResponse = await handleChatCompletions(chatReq, config, chatLogCtx); + await chatResponse.text(); + + // Claude Messages: Anthropic messages[] shape + const claudeLogCtx: RequestLogContext = { model: "", provider: "" }; + const claudeReq = new Request("http://localhost/v1/messages", { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": "fixture-key", + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: MODEL, + max_tokens: 128, + messages: [{ + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "AA==", + }, + }, + ], + }], + tools: [{ + name: "inspect", + input_schema: { type: "object", properties: {} }, + }], + }), + }); + const claudeResponse = await handleClaudeMessages(claudeReq, config, claudeLogCtx); + await claudeResponse.text(); + + // All three surfaces should select the same provider/model + expect(responsesLogCtx.provider).toBe("a"); + expect(responsesLogCtx.model).toBe("m1"); + expect(chatLogCtx.provider).toBe("a"); + expect(chatLogCtx.model).toBe("m1"); + expect(claudeLogCtx.provider).toBe("a"); + expect(claudeLogCtx.model).toBe("m1"); + + // All three surfaces should report satisfied requirements for tools and image + expect(responsesLogCtx.routeDecision).toBeDefined(); + expect(chatLogCtx.routeDecision).toBeDefined(); + expect(claudeLogCtx.routeDecision).toBeDefined(); + + const responsesToolsReq = responsesLogCtx.routeDecision!.requirements.find(r => r.id === "request-tools"); + const responsesImageReq = responsesLogCtx.routeDecision!.requirements.find(r => r.id === "request-image-input"); + expect(responsesToolsReq?.outcome).toBe("satisfied"); + expect(responsesImageReq?.outcome).toBe("satisfied"); + + const chatToolsReq = chatLogCtx.routeDecision!.requirements.find(r => r.id === "request-tools"); + const chatImageReq = chatLogCtx.routeDecision!.requirements.find(r => r.id === "request-image-input"); + expect(chatToolsReq?.outcome).toBe("satisfied"); + expect(chatImageReq?.outcome).toBe("satisfied"); + + const claudeToolsReq = claudeLogCtx.routeDecision!.requirements.find(r => r.id === "request-tools"); + const claudeImageReq = claudeLogCtx.routeDecision!.requirements.find(r => r.id === "request-image-input"); + expect(claudeToolsReq?.outcome).toBe("satisfied"); + expect(claudeImageReq?.outcome).toBe("satisfied"); + }); + + test("plain text with no tools produces no hard requirements on every surface", async () => { + adapterFactory = minimalSuccessAdapter; + const config = testConfig(); + + // Responses: simple text input + const responsesLogCtx: RequestLogContext = { model: "", provider: "" }; + const responsesReq = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: MODEL, + stream: false, + input: [{ + type: "message", + role: "user", + content: [{ type: "input_text", text: "hello" }], + }], + }), + }); + const responsesResponse = await handleResponses(responsesReq, config, responsesLogCtx); + await responsesResponse.text(); + + // Chat Completions: simple text message + const chatLogCtx: RequestLogContext = { model: "", provider: "" }; + const chatReq = new Request("http://localhost/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: MODEL, + stream: false, + messages: [{ role: "user", content: "hello" }], + }), + }); + const chatResponse = await handleChatCompletions(chatReq, config, chatLogCtx); + await chatResponse.text(); + + // Claude Messages: simple text message + const claudeLogCtx: RequestLogContext = { model: "", provider: "" }; + const claudeReq = new Request("http://localhost/v1/messages", { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": "fixture-key", + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: MODEL, + max_tokens: 128, + messages: [{ role: "user", content: "hello" }], + }), + }); + const claudeResponse = await handleClaudeMessages(claudeReq, config, claudeLogCtx); + await claudeResponse.text(); + + // All three surfaces should select the same provider/model + expect(responsesLogCtx.provider).toBe("a"); + expect(responsesLogCtx.model).toBe("m1"); + expect(chatLogCtx.provider).toBe("a"); + expect(chatLogCtx.model).toBe("m1"); + expect(claudeLogCtx.provider).toBe("a"); + expect(claudeLogCtx.model).toBe("m1"); + + // All three surfaces should have NO request-tools or request-image-input requirements + expect(responsesLogCtx.routeDecision).toBeDefined(); + expect(chatLogCtx.routeDecision).toBeDefined(); + expect(claudeLogCtx.routeDecision).toBeDefined(); + + const responsesHasTools = responsesLogCtx.routeDecision!.requirements.some(r => r.id === "request-tools"); + const responsesHasImage = responsesLogCtx.routeDecision!.requirements.some(r => r.id === "request-image-input"); + expect(responsesHasTools).toBe(false); + expect(responsesHasImage).toBe(false); + + const chatHasTools = chatLogCtx.routeDecision!.requirements.some(r => r.id === "request-tools"); + const chatHasImage = chatLogCtx.routeDecision!.requirements.some(r => r.id === "request-image-input"); + expect(chatHasTools).toBe(false); + expect(chatHasImage).toBe(false); + + const claudeHasTools = claudeLogCtx.routeDecision!.requirements.some(r => r.id === "request-tools"); + const claudeHasImage = claudeLogCtx.routeDecision!.requirements.some(r => r.id === "request-image-input"); + expect(claudeHasTools).toBe(false); + expect(claudeHasImage).toBe(false); + }); +}); From 73c2c14ede91211e3d7dae8e7d5e502e47996aae Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:54:55 +0200 Subject: [PATCH 13/77] fix(routing): preserve distinct policy fallback attempts --- src/server/responses/policy-fallback.ts | 50 ++++++++++++------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/server/responses/policy-fallback.ts b/src/server/responses/policy-fallback.ts index 5a701c6f4..fd63b40c3 100644 --- a/src/server/responses/policy-fallback.ts +++ b/src/server/responses/policy-fallback.ts @@ -1,7 +1,7 @@ import { comboFailureDecision } from "../../combos/failover"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { readJsonRequestBody } from "../request-decompress"; -import type { RequestLogContext } from "../request-log"; +import { finishRequestAttempt, type RequestLogContext } from "../request-log"; import type { OcxConfig } from "../../types"; import type { RouteCandidateTrace, RouteDecisionTraceV1 } from "../../routing/trace"; import { handleResponses as handleResponsesCore } from "./core"; @@ -46,18 +46,13 @@ function requestWithCandidate( candidate: Pick, ): Request { const headers = new Headers(req.headers); - // The retry body is re-serialized JSON. Carrying the original compression or - // byte length would make the child request malformed. headers.delete("content-encoding"); headers.delete("content-length"); headers.set("content-type", "application/json"); return new Request(req.url, { method: req.method, headers, - body: JSON.stringify({ - ...rawBody, - model: `${candidate.provider}/${candidate.model}`, - }), + body: JSON.stringify({ ...rawBody, model: `${candidate.provider}/${candidate.model}` }), signal: req.signal, }); } @@ -65,10 +60,7 @@ function requestWithCandidate( function errorCodeFromText(text: string): string | undefined { if (!text) return undefined; try { - const payload = JSON.parse(text) as { - error?: { code?: unknown; type?: unknown }; - code?: unknown; - }; + const payload = JSON.parse(text) as { error?: { code?: unknown; type?: unknown }; code?: unknown }; const candidate = payload.error?.code ?? payload.error?.type ?? payload.code; return typeof candidate === "string" ? candidate : undefined; } catch { @@ -77,15 +69,12 @@ function errorCodeFromText(text: string): string | undefined { } async function shouldHopPolicyCandidate(response: Response, signal?: AbortSignal): Promise { - if (response.status < 400) return false; + if (response.status < 400 || signal?.aborted) return false; try { const inspected = await readBoundedResponseBody(response.clone(), { signal }); const text = inspected.displaySafe ? inspected.text : ""; - return comboFailureDecision(response.status, text, { - code: errorCodeFromText(text), - }) === "hop"; + return comboFailureDecision(response.status, text, { code: errorCodeFromText(text) }) === "hop"; } catch { - // If the error body cannot be inspected safely, do not invent a retry. return false; } } @@ -94,6 +83,22 @@ function isPolicyDecision(trace: RouteDecisionTraceV1 | undefined): trace is Rou return trace?.routeKind === "policy" && !!trace.profile; } +/** Finalize the failed physical attempt so the retry receives a fresh attempt row. */ +function finishFailedPolicyAttempt(logCtx: RequestLogContext, status: number): void { + const attempt = logCtx.activeAttempt; + if (attempt) { + const startedAt = logCtx.activeAttemptStartedAt ?? Date.now(); + finishRequestAttempt(attempt, status, Math.max(0, Date.now() - startedAt), attempt.usage ?? logCtx.usage); + } + delete logCtx.activeAttempt; + delete logCtx.activeAttemptStartedAt; + delete logCtx.usage; + delete logCtx.usageFromBridge; + delete logCtx.upstreamError; + delete logCtx.terminalHttpStatus; + delete logCtx.terminalIncompleteReason; +} + /** * Run a Responses request and, only for an explicitly selected policy profile, * hop to the next eligible policy candidate after a retryable pre-success @@ -108,16 +113,10 @@ export async function handleResponsesWithPolicyFallback( deps: PolicyFallbackDeps = {}, ): Promise { const runCore = deps.runCore ?? handleResponsesCore; - - // Capture a replayable, decompressed body before the core consumes the - // request. If decoding fails, defer entirely to the canonical core path so - // its existing error semantics remain unchanged. let rawBody: Record | null = null; try { const parsed = await readJsonRequestBody(req.clone()); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - rawBody = parsed as Record; - } + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) rawBody = parsed as Record; } catch { // Core owns the client-facing parse/decompression error. } @@ -137,14 +136,11 @@ export async function handleResponsesWithPolicyFallback( if (!next) return response; tried.add(candidateKey(next)); + finishFailedPolicyAttempt(logCtx, response.status); const retryRequest = requestWithCandidate(req, rawBody, next); try { response = await runCore(retryRequest, config, logCtx, options); } finally { - // A fallback child routes explicitly and therefore produces its own - // explicit-provider trace. Keep the original policy decision as the - // request-level WHY while retaining the child's physical model/provider - // and attempts on the mutable log context. logCtx.requestedModel = initialRequestedModel; logCtx.routeDecision = initialTrace; } From f8a683b28490eece886c1db0e8bebdc0f5ac0c83 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:55:31 +0200 Subject: [PATCH 14/77] test(routing): cover fallback attempt and no-hop guards --- tests/routing-policy-fallback.test.ts | 182 ++++++++++++-------------- 1 file changed, 85 insertions(+), 97 deletions(-) diff --git a/tests/routing-policy-fallback.test.ts b/tests/routing-policy-fallback.test.ts index 11e9360d9..26503e5eb 100644 --- a/tests/routing-policy-fallback.test.ts +++ b/tests/routing-policy-fallback.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { OcxConfig } from "../src/types"; -import type { RequestLogContext } from "../src/server/request-log"; +import { beginRequestAttempt, type RequestLogContext } from "../src/server/request-log"; import type { RouteDecisionTraceV1 } from "../src/routing/trace"; import { handleResponsesWithPolicyFallback, @@ -18,138 +18,126 @@ function policyTrace(): RouteDecisionTraceV1 { profile: { id: "daily", revision: "rev-1" }, requirements: [], candidates: [ - { - provider: "provider-a", - model: "model-a", - eligible: true, - exclusions: [], - score: { total: 0.90, components: {} }, - }, - { - provider: "provider-b", - model: "model-b", - eligible: true, - exclusions: [], - score: { total: 0.80, components: {} }, - }, - { - provider: "provider-c", - model: "model-c", - eligible: true, - exclusions: [], - score: { total: 0.80, components: {} }, - }, - { - provider: "provider-d", - model: "model-d", - eligible: false, - exclusions: [{ code: "tools" }], - score: { total: 1, components: {} }, - }, + { provider: "provider-a", model: "model-a", eligible: true, exclusions: [], score: { total: 0.90, components: {} } }, + { provider: "provider-b", model: "model-b", eligible: true, exclusions: [], score: { total: 0.80, components: {} } }, + { provider: "provider-c", model: "model-c", eligible: true, exclusions: [], score: { total: 0.80, components: {} } }, + { provider: "provider-d", model: "model-d", eligible: false, exclusions: [{ code: "tools" }], score: { total: 1, components: {} } }, ], - selected: { - candidateIndex: 0, - provider: "provider-a", - model: "model-a", - reason: "highest-score", - }, + selected: { candidateIndex: 0, provider: "provider-a", model: "model-a", reason: "highest-score" }, }; } -function request(): Request { +function request(signal?: AbortSignal): Request { return new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "policy/daily", input: "hello", stream: false }), + signal, }); } +function seedAttempt(logCtx: RequestLogContext, provider: string, model: string): void { + if (logCtx.activeAttempt) return; + const attempt = beginRequestAttempt((logCtx.attempts?.length ?? 0) + 1, provider, model, "test"); + (logCtx.attempts ??= []).push(attempt); + logCtx.activeAttempt = attempt; + logCtx.activeAttemptStartedAt = Date.now(); +} + describe("policy candidate fallback", () => { test("ranks only eligible untried candidates by score and stable original order", () => { - const trace = policyTrace(); - const ranked = rankPolicyFallbackCandidates( - trace, - new Set(["provider-a\u0000model-a"]), - ); - + const ranked = rankPolicyFallbackCandidates(policyTrace(), new Set(["provider-a\u0000model-a"])); expect(ranked.map(candidate => `${candidate.provider}/${candidate.model}`)).toEqual([ "provider-b/model-b", "provider-c/model-c", ]); }); - test("retries the next policy candidate after a retryable pre-stream failure", async () => { + test("retries the next policy candidate and keeps distinct physical attempts", async () => { const trace = policyTrace(); - const logCtx = { - requestedModel: "policy/daily", - routeDecision: trace, - attempts: [], - } as unknown as RequestLogContext; + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; const seenModels: string[] = []; - const response = await handleResponsesWithPolicyFallback( - request(), - {} as OcxConfig, - logCtx, - {}, - { - runCore: async (req, _config, childLog) => { - const body = await req.json() as { model: string }; - seenModels.push(body.model); - if (seenModels.length === 1) { - childLog.requestedModel = "policy/daily"; - childLog.routeDecision = trace; - return new Response( - JSON.stringify({ error: { message: "rate limited", type: "rate_limit_error" } }), - { status: 429, headers: { "content-type": "application/json" } }, - ); - } - childLog.requestedModel = body.model; - childLog.routeDecision = { - ...trace, - requestedModel: body.model, - routeKind: "explicit-provider", - profile: undefined, - }; - return new Response(JSON.stringify({ status: "completed" }), { status: 200 }); - }, + const response = await handleResponsesWithPolicyFallback(request(), {} as OcxConfig, logCtx, {}, { + runCore: async (req, _config, childLog) => { + const body = await req.json() as { model: string }; + seenModels.push(body.model); + const first = seenModels.length === 1; + seedAttempt(childLog, first ? "provider-a" : "provider-b", first ? "model-a" : "model-b"); + if (first) { + childLog.requestedModel = "policy/daily"; + childLog.routeDecision = trace; + return new Response(JSON.stringify({ error: { message: "rate limited", type: "rate_limit_error" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + childLog.requestedModel = body.model; + childLog.routeDecision = { ...trace, requestedModel: body.model, routeKind: "explicit-provider", profile: undefined }; + return new Response(JSON.stringify({ status: "completed" }), { status: 200 }); }, - ); + }); expect(response.status).toBe(200); expect(seenModels).toEqual(["policy/daily", "provider-b/model-b"]); expect(logCtx.requestedModel).toBe("policy/daily"); expect(logCtx.routeDecision).toBe(trace); + expect(logCtx.attempts).toHaveLength(2); + expect(logCtx.attempts?.[0]).toMatchObject({ provider: "provider-a", model: "model-a", status: 429 }); + expect(logCtx.attempts?.[1]).toMatchObject({ provider: "provider-b", model: "model-b" }); + expect(logCtx.activeAttempt).toBe(logCtx.attempts?.[1]); }); test("does not switch candidates for terminal client/input failures", async () => { const trace = policyTrace(); - const logCtx = { - requestedModel: "policy/daily", - routeDecision: trace, - attempts: [], - } as unknown as RequestLogContext; + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; let calls = 0; + const response = await handleResponsesWithPolicyFallback(request(), {} as OcxConfig, logCtx, {}, { + runCore: async (_req, _config, childLog) => { + calls += 1; + childLog.requestedModel = "policy/daily"; + childLog.routeDecision = trace; + return new Response(JSON.stringify({ error: { message: "invalid request", type: "invalid_request_error" } }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + }, + }); + expect(response.status).toBe(400); + expect(calls).toBe(1); + }); - const response = await handleResponsesWithPolicyFallback( - request(), - {} as OcxConfig, - logCtx, - {}, - { - runCore: async (_req, _config, childLog) => { - calls += 1; - childLog.requestedModel = "policy/daily"; - childLog.routeDecision = trace; - return new Response( - JSON.stringify({ error: { message: "invalid request", type: "invalid_request_error" } }), - { status: 400, headers: { "content-type": "application/json" } }, - ); - }, + test("does not switch candidates after client cancellation", async () => { + const trace = policyTrace(); + const controller = new AbortController(); + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + let calls = 0; + const response = await handleResponsesWithPolicyFallback(request(controller.signal), {} as OcxConfig, logCtx, {}, { + runCore: async (_req, _config, childLog) => { + calls += 1; + childLog.routeDecision = trace; + controller.abort(); + return new Response(JSON.stringify({ error: { type: "rate_limit_error" } }), { status: 429 }); }, - ); + }); + expect(response.status).toBe(429); + expect(calls).toBe(1); + }); - expect(response.status).toBe(400); + test("does not switch candidates after a streaming response has started", async () => { + const trace = policyTrace(); + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + let calls = 0; + const body = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.failed\"}\n\n"; + const response = await handleResponsesWithPolicyFallback(request(), {} as OcxConfig, logCtx, {}, { + runCore: async (_req, _config, childLog) => { + calls += 1; + childLog.routeDecision = trace; + return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }); + }, + }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("hello"); expect(calls).toBe(1); }); }); From 9d7a581515bd51170f3e74c867b6ede860051e5a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:56:25 +0200 Subject: [PATCH 15/77] fix(routing): keep percentless Codex quota unknown --- src/routing/quota.ts | 67 ++++++++------------------------------------ 1 file changed, 11 insertions(+), 56 deletions(-) diff --git a/src/routing/quota.ts b/src/routing/quota.ts index 3d0bd6bd7..c239980e2 100644 --- a/src/routing/quota.ts +++ b/src/routing/quota.ts @@ -24,11 +24,8 @@ import type { RouteQuotaEvidence } from "./trace"; export interface QuotaEvidenceInput { provider: string; model: string; - /** Opaque account reference for per-account quota sources. */ accountRef?: string; - /** Codex pool account id (provider "openai"). */ codexAccountId?: string; - /** The account's plan (provider "openai"); selects the governing quota window. */ codexAccountPlan?: string; } @@ -40,16 +37,15 @@ export interface CodexPoolQuotaAccount { function codexAccountQuotaEvidence(accountId: string, plan?: string): RouteQuotaEvidence { const quota = getAccountQuota(accountId); if (!quota) return { known: false }; - - // Go/Free accounts report a 30-day window only; weekly windows gate - // everything else. `codexQuotaWindowForPlan` is the single shared rule - // (parser, exhaustion, recovery), so select the plan-specific bars here too. const monthly = codexQuotaWindowForPlan(plan) === "monthly"; const percents = [ ...(monthly ? [] : [quota.weeklyPercent]), quota.monthlyPercent, ].filter((value): value is number => typeof value === "number" && Number.isFinite(value)); const maxPercent = percents.length > 0 ? Math.max(...percents) : undefined; + // Credits-only snapshots prove neither usage nor exhaustion. Unknown must not + // become healthy capacity merely because a cache row exists. + if (maxPercent === undefined) return { known: false }; const resets = [ ...(monthly ? [] : [quota.weeklyResetAt]), quota.monthlyResetAt, @@ -57,32 +53,21 @@ function codexAccountQuotaEvidence(accountId: string, plan?: string): RouteQuota .filter(value => value > Date.now()); return { known: true, - ...(maxPercent !== undefined - ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } - : {}), + headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)), exhausted: isCodexQuotaExhausted(quota, plan), ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), source: "codex-pool", }; } -/** - * Provider-level quota evidence for a Codex account pool. A policy profile - * chooses provider/model, while the existing pool remains authoritative for - * the physical account. Therefore the provider is usable when ANY known pool - * account has headroom. Unknown accounts prevent a known-exhausted verdict: - * unknown capacity is not zero capacity. - */ export function codexPoolQuotaEvidence(accounts: readonly CodexPoolQuotaAccount[]): RouteQuotaEvidence { if (accounts.length === 0) return { known: false }; const evidence = accounts.map(account => codexAccountQuotaEvidence(account.accountId, account.plan)); const known = evidence.filter(item => item.known); if (known.length === 0) return { known: false }; - const usable = known.filter(item => item.exhausted !== true); if (usable.length > 0) { - const headrooms = usable - .map(item => item.headroom) + const headrooms = usable.map(item => item.headroom) .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); return { known: true, @@ -91,13 +76,8 @@ export function codexPoolQuotaEvidence(accounts: readonly CodexPoolQuotaAccount[ source: "codex-pool", }; } - - // At least one account has no quota evidence. The pool may still be usable, - // so fail open as unknown rather than excluding the provider as exhausted. if (known.length < evidence.length) return { known: false }; - - const resets = known - .map(item => item.resetAtMs) + const resets = known.map(item => item.resetAtMs) .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); return { known: true, @@ -108,16 +88,11 @@ export function codexPoolQuotaEvidence(accounts: readonly CodexPoolQuotaAccount[ }; } -/** - * Assemble quota evidence from canonical local caches only (no network). - * Unknown dimensions stay unknown - never zero. - */ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuotaEvidence { if (input.provider === "openai" && input.codexAccountId) { - // The live routing/profile assembly path includes the selected account's - // plan. At that boundary the candidate represents the whole Codex pool, - // not that one active account, so aggregate the reconciled quota cache. - // Caller-supplied dry-run account evidence omits the plan and remains exact. + // listAccountQuotas() is the reconciled quota snapshot: config-generation + // reconciliation prunes removed accounts. Other eligibility dimensions + // (pause/reauth/cooldown/soft-avoid) remain health/pool-selector concerns. if (input.codexAccountPlan !== undefined) { const pool = [...listAccountQuotas()].map(([accountId]) => ({ accountId, @@ -131,10 +106,6 @@ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuota if (input.provider === "anthropic" && input.accountRef) { const quota = getCachedProviderAccountQuota("anthropic", input.accountRef); if (quota) { - // Anthropic per-family buckets (e.g. Opus / Sonnet) are stricter than the - // broad account windows: fold the candidate model's matching bucket into - // headroom and exhaustion so a model-specific overage is not hidden by - // a healthy aggregate window. const family = anthropicFamilyWindow(input.model, quota.customWindows ?? []); const percents = [quota.fiveHourPercent, quota.weeklyPercent, quota.monthlyPercent, family?.percent] .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); @@ -144,25 +115,16 @@ export function quotaEvidenceForCandidate(input: QuotaEvidenceInput): RouteQuota .filter(value => value > Date.now()); return { known: true, - ...(maxPercent !== undefined - ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } - : {}), + ...(maxPercent !== undefined ? { headroom: Math.max(0, Math.min(1, 1 - maxPercent / 100)) } : {}), exhausted: maxPercent !== undefined && maxPercent >= 100, ...(resets.length > 0 ? { resetAtMs: Math.min(...resets) } : {}), source: "provider-report", }; } } - return { known: false }; } -/** - * Match the candidate model to an Anthropic per-family quota window. Window - * labels from the provider probe are "Opus" / "Sonnet"; model ids carry the - * family as a segment (e.g. `claude-opus-...`, `claude-sonnet-...`). Returns - * undefined when no family window is cached or no label matches. - */ function anthropicFamilyWindow( model: string, windows: Array<{ label: string; percent?: number; resetAt?: number }>, @@ -170,18 +132,11 @@ function anthropicFamilyWindow( const normalized = model.toLowerCase(); for (const window of windows) { const family = window.label.trim().toLowerCase(); - if (family && normalized.includes(family)) { - return window; - } + if (family && normalized.includes(family)) return window; } return undefined; } -/** - * Deterministic quota score in [0,1]: larger available headroom scores - * higher; exhausted evidence scores 0. Unknown evidence returns null so the - * caller can apply the profile's unknownEvidence policy. - */ export function quotaScore(evidence: RouteQuotaEvidence | undefined): number | null { if (!evidence || !evidence.known) return null; if (evidence.exhausted === true) return 0; From b31abddddd809f9e61c6130efd7846655a663c04 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:57:03 +0200 Subject: [PATCH 16/77] test(routing): keep credits-only pool quota unknown --- tests/routing-policy-pool-quota.test.ts | 30 ++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/tests/routing-policy-pool-quota.test.ts b/tests/routing-policy-pool-quota.test.ts index 24dea36b0..89f3ce1d7 100644 --- a/tests/routing-policy-pool-quota.test.ts +++ b/tests/routing-policy-pool-quota.test.ts @@ -9,44 +9,30 @@ describe("Codex pool quota evidence for routing policies", () => { test("uses the best known usable headroom instead of only one active account", () => { setAccountQuotaFromParsed("low", { weeklyPercent: 95 }); setAccountQuotaFromParsed("healthy", { weeklyPercent: 20 }); - expect(codexPoolQuotaEvidence([ { accountId: "low", plan: "plus" }, { accountId: "healthy", plan: "plus" }, - ])).toMatchObject({ - known: true, - exhausted: false, - headroom: 0.8, - source: "codex-pool", - }); + ])).toMatchObject({ known: true, exhausted: false, headroom: 0.8, source: "codex-pool" }); }); test("the live policy evidence path aggregates the reconciled pool", () => { setAccountQuotaFromParsed("active", { weeklyPercent: 96 }); setAccountQuotaFromParsed("alternate", { weeklyPercent: 25 }); - expect(quotaEvidenceForCandidate({ provider: "openai", model: "gpt-5.6", codexAccountId: "active", codexAccountPlan: "plus", - })).toMatchObject({ - known: true, - exhausted: false, - headroom: 0.75, - source: "codex-pool", - }); + })).toMatchObject({ known: true, exhausted: false, headroom: 0.75, source: "codex-pool" }); }); test("reports exhausted only when every pool account is known exhausted", () => { setAccountQuotaFromParsed("a", { weeklyPercent: 100, weeklyResetAt: Date.now() + 60_000 }); setAccountQuotaFromParsed("b", { weeklyPercent: 100, weeklyResetAt: Date.now() + 120_000 }); - const evidence = codexPoolQuotaEvidence([ { accountId: "a", plan: "plus" }, { accountId: "b", plan: "plus" }, ]); - expect(evidence.known).toBe(true); expect(evidence.exhausted).toBe(true); expect(evidence.headroom).toBe(0); @@ -55,10 +41,18 @@ describe("Codex pool quota evidence for routing policies", () => { test("does not call a partially unknown pool exhausted", () => { setAccountQuotaFromParsed("known-exhausted", { weeklyPercent: 100 }); - expect(codexPoolQuotaEvidence([ { accountId: "known-exhausted", plan: "plus" }, { accountId: "unknown", plan: "plus" }, ])).toEqual({ known: false }); }); -}); \ No newline at end of file + + test("credits-only cached evidence stays unknown", () => { + setAccountQuotaFromParsed("known-exhausted", { weeklyPercent: 100 }); + setAccountQuotaFromParsed("credits-only", { resetCredits: 7 }); + expect(codexPoolQuotaEvidence([ + { accountId: "known-exhausted", plan: "plus" }, + { accountId: "credits-only", plan: "plus" }, + ])).toEqual({ known: false }); + }); +}); From e2ea3312f83e00c0ce4c2120aaa2f86d7df2b6e4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:49:10 +0200 Subject: [PATCH 17/77] docs(lab): define compatibility evidence contracts Freeze the CL-00 evidence, verdict, scenario, identity, privacy, and incident contracts so conformance implementation can proceed without duplicating routing authority. --- .../000_master_plan.md | 303 ++++++++++ .../001_pr_stack_status.md | 72 +++ .../010_architecture_and_evidence_contract.md | 549 ++++++++++++++++++ .../020_scenario_contract_and_catalogue.md | 394 +++++++++++++ .../021_protocol_v1_manifest_authority.md | 342 +++++++++++ .../022_protocol_v1_cases.json | 456 +++++++++++++++ .../030_incident_corpus.md | 457 +++++++++++++++ .../040_security_and_privacy.md | 239 ++++++++ .../050_cl00_acceptance_review.md | 147 +++++ 9 files changed, 2959 insertions(+) create mode 100644 devlog/_plan/260807_compatibility_lab/000_master_plan.md create mode 100644 devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md create mode 100644 devlog/_plan/260807_compatibility_lab/010_architecture_and_evidence_contract.md create mode 100644 devlog/_plan/260807_compatibility_lab/020_scenario_contract_and_catalogue.md create mode 100644 devlog/_plan/260807_compatibility_lab/021_protocol_v1_manifest_authority.md create mode 100644 devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json create mode 100644 devlog/_plan/260807_compatibility_lab/030_incident_corpus.md create mode 100644 devlog/_plan/260807_compatibility_lab/040_security_and_privacy.md create mode 100644 devlog/_plan/260807_compatibility_lab/050_cl00_acceptance_review.md diff --git a/devlog/_plan/260807_compatibility_lab/000_master_plan.md b/devlog/_plan/260807_compatibility_lab/000_master_plan.md new file mode 100644 index 000000000..fa41e24b4 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/000_master_plan.md @@ -0,0 +1,303 @@ +# OpenCodex Compatibility Lab / EvalGrid + +Status: CL-00 architecture authority +Authority baseline: `upstream/dev` at `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` +Package/runtime at baseline: OpenCodex `2.10.2`, Bun `1.3.14` + +## Purpose + +Compatibility Lab turns compatibility claims into bounded, reproducible +evidence. It tests OpenCodex protocol behavior, exact configured routes, and +later execution-grounded task outcomes without becoming a provider registry, a +user-policy system, or a production router. + +This directory is the programme authority. The contracts frozen by CL-00 are: + +- [Architecture and evidence](./010_architecture_and_evidence_contract.md) +- [Scenario model and initial catalogue](./020_scenario_contract_and_catalogue.md) +- [Protocol V1 manifest authority](./021_protocol_v1_manifest_authority.md) +- [Protocol V1 canonical cases](./022_protocol_v1_cases.json) +- [Historical incident corpus](./030_incident_corpus.md) +- [Security and privacy](./040_security_and_privacy.md) +- [CL-00 independent acceptance review](./050_cl00_acceptance_review.md) +- [PR stack status](./001_pr_stack_status.md) + +Later phases may add implementation detail, but must amend these contracts +explicitly rather than silently changing their meaning. + +## Live repository truth + +CL-00 audited the live `dev` tree before defining new authority. + +### Shipped and authoritative + +- Provider declarations and model metadata: + `src/providers/registry.ts`, `src/providers/derive.ts`, `src/types.ts`; + route-time claimed capability assembly in `src/routing/capability.ts` also + consumes provider config, cached Codex catalog rows and native metadata. + Generated fallback metadata lives in `src/generated/model-metadata.ts` and is + sourced by `scripts/model-metadata.source.json`. +- Routing Profile public/config types: `OcxRoutingProfileConfig` in + `src/types.ts`; validation, normalization, revision hashing, persistence and + resolution in `src/routing/profile.ts`. +- Deterministic profile evaluation and route traces: + `src/routing/evaluator.ts`, `src/routing/trace.ts`, and + `src/router.ts`. +- Profile management CRUD and dry-run: + `src/server/management/routing-profile-routes.ts`. +- Dashboard profile editor, dry-run and routing analytics: + `gui/src/pages/RoutingProfiles.tsx`, mounted under Models -> Routing. +- Immutable request/usage evidence and rebuildable history projection: + `src/usage/log.ts`, `src/routing/history/indexer.ts`, and + `src/routing/analytics.ts`. +- Why-this-route evidence: + `RouteDecisionTraceV1`, request-history explain endpoints in + `src/server/management/request-history-routes.ts`, and CLI explain support. + The GUI Logs modal renders only a compact route summary through + `gui/src/pages/log-route-decision.ts`; there is no GUI request-history browser + or full trace + attempts + outcome view on this baseline. +- Existing diagnostics are narrower than Compatibility Lab: + `ocx doctor` is observe-only environment/OAuth/runtime diagnosis, while + `POST /api/providers/test` performs a bounded live `/models` connectivity + check only when applicable; forward providers return configured status and + static catalogues return not-applicable without network access. +- Protocol behavior is already covered by many focused tests under `tests/`, + but those tests are not a versioned scenario catalogue or evidence ledger. + +### Not shipped + +- No Compatibility Lab runner, scenario registry, evidence ledger, SQLite + projection, CLI, management API, or UI exists. +- Generated Cursor agent protobufs include task/grind/subagent message types, + but OpenCodex has no native Agent Fabric task persistence, harness handoff, + portable task-state model, or management API. Router Intelligence's own + master plan explicitly excluded Agent Fabric. +- No Routing Profile compatibility fields exist. + +Current routing nuances that later phases must preserve rather than +over-describe: + +- selection traces and execution attempts are separate; the explain API merges + trace + `attempts[]` + final outcome at read time; +- `optimize.latency` is currently a declaration-priority share, while observed + latency contributes through health evidence rather than an independent + top-level score; +- cost evidence is commonly unknown on the live pre-dispatch path because + request usage is not yet available; +- profile dry-run is evaluation-only and never dispatches upstream; +- an unknown canonical `policy/` currently falls through to ordinary model + routing rather than failing closed. + +Consequently, CL-00 defines future contracts and integration seams only. It +does not rename existing Router Intelligence concepts or describe speculative +Agent Fabric endpoints as current behavior. + +## Architectural invariant + +```text +Provider Registry + ↓ +Compatibility Lab + ↓ +Compatibility Graph / Verified Evidence + ↓ +Routing Profiles + ↓ +Router Intelligence + ↓ +Selected Model / Route + ↓ +Agent Fabric / Real Execution + └──────────────→ execution-grounded outcomes back to Lab +``` + +The arrows are data dependencies, not ownership transfers. + +### Provider Registry + +The Provider Registry declares what a provider/model is believed to support and +supplies defaults used to construct an effective route. The shipped +`candidateCapabilityEvidence()` also combines explicit provider config, cached +catalog rows, adapter-level inference and native-model metadata. These local +declarations are claims. They may seed `CLAIMED`; they cannot by themselves +produce `PROBED`, `VERIFIED`, `DEGRADED`, or `UNSUPPORTED`. + +The Lab may snapshot a registry claim with its source revision for +reproducibility. It must not create a parallel provider catalogue or write +provider declarations back into the registry. + +Registry-owned runtime defaults such as model wire selection, discovery policy, +upstream streaming, reasoning replay, service-tier support, and item-ID repair +are intentionally not all persisted to `config.json`. Claim snapshots capture +the effective sources; they do not freeze runtime defaults into user config. + +### Compatibility Lab + +The Lab owns versioned scenarios, immutable compatibility evidence, failure +attribution, freshness, derived verdicts, and regression history. It may +project evidence into a compatibility graph keyed by exact route subject and +evidence layer. + +The Lab never chooses a production candidate, mutates a Routing Profile, +changes provider metadata, or turns a probe result directly into a route. + +### Routing Profiles + +Routing Profiles remain the only user-policy layer. Future compatibility +requirements extend `OcxRoutingProfileConfig`, its normalizer/revision, the +existing evaluator, the existing management CRUD/dry-run endpoints, and the +Models dashboard editor. There will be no compatibility-specific profile +store, evaluator, or editor. + +### Router Intelligence + +Router Intelligence combines the selected profile with current capability, +compatibility, health, quota, cost, and latency evidence and makes the +deterministic route decision. Its existing `RouteDecisionTraceV1` remains the +authority for explaining that decision. Compatibility inputs will later add +bounded evidence to that trace rather than introduce a second explanation +record. + +### Agent Fabric + +Agent Fabric is a future producer of task-effectiveness observations. It owns +real task execution and its sandbox. The Lab accepts only structured outcome +data and sanitized content-addressed artifact references; it does not copy task +repositories, prompts, worktrees, or hidden reasoning. + +Because a native Agent Fabric is not present on the CL-00 baseline, this +programme freezes the consumer semantics, not a fictitious production API. A +later producer contract must identify its schema version, task class, exact +route subject, deterministic verifier results, timing, resource limits, +outcome, and sanitized artifact references. Existing request-grounded evidence +may be linked through `RouteDecisionTraceV1`, `PersistedUsageAttempt`, and the +final request outcome; prompt-bearing `responses-state.json` and generated +Cursor task protobufs are not Lab feeds. + +## Evidence-layer invariant + +Every scenario and observation has exactly one layer: + +1. `protocol_conformance`: whether OpenCodex translates and preserves a + protocol contract correctly. +2. `live_route_compatibility`: whether an exact + provider/model/adapter/configuration route works now. +3. `task_effectiveness`: whether that route produces verifier-confirmed + outcomes for a versioned class of coding work. + +Verdicts are projected per `(subject, layer, suite)`. Evidence from one layer +may be shown as a prerequisite or correlated signal, but cannot promote or +degrade another layer's verdict. There is no universal compatibility score. + +## Persistence authority + +Future implementation uses the existing OpenCodex config root returned by +`getConfigDir()` (`OPENCODEX_HOME`, default `~/.opencodex`) and owns: + +```text +~/.opencodex/lab/ + compatibility.jsonl + compatibility.sqlite + artifacts/ +``` + +- `compatibility.jsonl` is the canonical append-only evidence/event ledger. +- `compatibility.sqlite` is a disposable query projection rebuilt from JSONL. +- `artifacts/` contains bounded, sanitized, content-addressed artifacts. +- Scenario/suite manifests and synthetic fixture/source anchors are + content-addressed contract artifacts retained with the observations that + reference them. +- Verdicts are derived projections, never mutable canonical booleans. +- Corrections append invalidation/supersession events; prior bytes are not + edited. +- The Lab does not copy `usage.jsonl` or routing-history rows. When useful, an + observation references an existing request ID or route decision ID. +- Agent Fabric supplies structured outcome data/references, never repositories + or prompt transcripts. + +This location follows current repository state-root conventions. No filename +or location change from the proposed architecture was justified by the audit. + +## Routing Profiles boundary for CL-06 + +CL-06 must add optional compatibility controls alongside existing capability, +health, quota, cost, and latency policy: + +- required compatibility suites; +- minimum compatibility status; +- maximum evidence age; +- unknown-evidence behavior; +- degraded-evidence behavior. + +`minimum compatibility status` is not a total ordering across all verdicts. +Only `PROBED` and `VERIFIED` are positive thresholds. `DEGRADED` is governed by +its explicit behavior, `UNKNOWN`/`CLAIMED`/`BLOCKED` by unknown-evidence +behavior, and `UNSUPPORTED` fails a required suite. + +The exact future flow is: + +```text +Routing Profile + ↓ +Configured candidates + ↓ +Hard capability gates + ↓ +Compatibility requirements / penalties + ↓ +Eligible candidates + ↓ +Health / quota / cost / latency scoring + ↓ +Deterministic winner +``` + +All compatibility fields are optional. Profiles that omit them retain their +current validation, revision, eligibility and scoring behavior. A profile +evaluation reads an existing projection only. No compatibility probe, network +request, task, or projection rebuild may run synchronously on the production +request path. + +## Programme phases + +Only CL-00 is authorized by this document at present. + +| Phase | Purpose | Authorization | +|---|---|---| +| CL-00 | Architecture authority, contracts, scenario catalogue, incident corpus | This PR | +| CL-01 | Deterministic protocol-conformance runner and fixtures | Not started; requires accepted CL-00 | +| CL-02 | Immutable JSONL ledger, artifacts and SQLite projection | Not started | +| CL-03 | Bounded live-route probes | Not started | +| CL-04 | Lab CLI and management read surfaces | Not started | +| CL-05 | Compatibility Matrix UI | Not started | +| CL-06 | Existing Routing Profile compatibility controls and Router Intelligence consumption | Not started | +| CL-07 | Agent Fabric task-effectiveness ingestion | Not started | +| CL-08 | Shadow/automatic/public evidence workflows | Not started | + +Phase numbering after CL-01 is programme planning, not implementation +authorization. A later accepted plan may split a phase while preserving these +ownership boundaries. + +## CL-00 acceptance criteria + +CL-00 is accepted only when: + +1. all three evidence layers have separate subjects, scenarios and verdicts; +2. every canonical verdict is reproducible from immutable inputs; +3. environmental blockers cannot poison compatibility conclusions; +4. exact route identity prevents evidence reuse across behavior changes; +5. scenario semantics and initial IDs are implementable without an LLM judge; +6. representative historical incidents map to abstract regression scenarios; +7. future compatibility policy extends existing Routing Profiles; +8. probes and task execution are excluded from production request routing; +9. privacy and sandbox ceilings are explicit; +10. an independent review finds no unresolved Critical, High, or Medium issue. + +## CL-00 hard stop + +This phase does not implement a runner, mock upstream, persistence code, live +probe, CLI, management endpoint, UI, profile field, routing behavior, shadow +route, Fabric ingestion, automatic routing, or public publisher. + +Acceptance of CL-00 authorizes discussion and planning of CL-01; it does not +start CL-01 automatically. diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md new file mode 100644 index 000000000..e084fa66a --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -0,0 +1,72 @@ +# Compatibility Lab PR stack status + +Updated throughout the programme. Every phase records its branch, exact base +and implementation head, PR, verification, independent review, blockers, and +whether the next phase is authorized. + +## Programme facts + +- Repository: `lidge-jun/opencodex` +- Integration target: `dev` +- CL-00 starting `upstream/dev`: + `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` +- Package/runtime at start: OpenCodex `2.10.2`, Bun `1.3.14` +- CL-00 branch: `feat/cl-00-compatibility-contracts` +- CL-00 scope: documentation/contracts/incident corpus only +- PR target: `lidge-jun/opencodex:dev` + +## Stack + +| Phase | Branch | Base SHA | Implementation head | PR | State | +|---|---|---|---|---|---| +| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | pending commit | pending | ACCEPTED | +| CL-01 | not created | CL-00 must be accepted first | not started | none | NOT AUTHORIZED | + +## CL-00 acceptance log + +- Live-tree audit: complete against the exact base above. Audited provider + registry/derivation, Routing Profile types/normalization/evaluator/API/UI/ + dry-run, route traces and Why-this-route, usage/request-history/analytics, + doctor/provider connectivity validation, protocol regression tests, and + relevant open/closed devlog incidents. +- Live-tree correction: generated Cursor task/grind protobuf messages exist, + but no native Agent Fabric task persistence or management contract and no + Compatibility Lab implementation exists. CL-00 freezes consumer semantics + without claiming production endpoints. Future compatibility policy must + extend the shipped Routing Profiles system. +- Documents: + - `000_master_plan.md` + - `010_architecture_and_evidence_contract.md` + - `020_scenario_contract_and_catalogue.md` + - `021_protocol_v1_manifest_authority.md` + - `022_protocol_v1_cases.json` + - `030_incident_corpus.md` + - `040_security_and_privacy.md` + - `050_cl00_acceptance_review.md` (recorded after independent review) +- Baseline verification on clean CL-00 worktree: + - `bun x tsc --noEmit`: passed, 0 errors. + - `bun run privacy:scan`: passed. + - `bun run test`: **not green** on this Windows/Bun 1.3.14 host. The full + run exited 3 after a cache-invalidation failure, an empty Windows + effective-account lookup, and a Bun + `panic(main thread): index out of bounds: index 0, len 0`. + - Serial isolation: + `tests/codex-models-cache-invalidate.test.ts` passed 6/6; + `tests/codex-native-residue.test.ts` passed 63 with 2 platform skips. + - Focused protocol/compatibility suite excluding privileged-symlink state + cases: 395 passed, 0 failed across 24 files. + - Focused continuation state semantics: 2 passed, 95 filtered, 0 failed. + - A broader focused run including all `responses-state.test.ts` cases had + 488 pass and 4 fail; all four failures were Windows `EPERM` creating + symlinks on this host. + - `tests/repo-hygiene.test.ts`: 11 passed, 0 failed. +- Documentation verification: complete. JSON authority parses; all 35 cases, + 46 fixture records, eight suites and fixture digests validate; all local + CL-00 links resolve; `git diff --check` passed. +- Independent acceptance review: accepted after correction; all ten required + challenges pass and no Critical/High/Medium findings remain. +- Blockers: none for CL-00. Full-suite green remains unavailable on this host + for the Windows/Bun reasons above. +- CL-00 ending implementation SHA: pending finalization. +- Draft PR: pending. +- CL-01 authorized: **NO**. Acceptance of CL-00 does not start CL-01. diff --git a/devlog/_plan/260807_compatibility_lab/010_architecture_and_evidence_contract.md b/devlog/_plan/260807_compatibility_lab/010_architecture_and_evidence_contract.md new file mode 100644 index 000000000..11c7ba60c --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/010_architecture_and_evidence_contract.md @@ -0,0 +1,549 @@ +# CL-00 architecture and evidence contract + +This document freezes the semantic model consumed by later Compatibility Lab +phases. Names shown in code blocks are contract names, not claims that +production TypeScript types already exist. + +## 1. Evidence layers + +### `protocol_conformance` + +Question: does this OpenCodex build preserve the declared inbound-to-upstream +and upstream-to-client protocol contract? + +Inputs are deterministic fixture requests and deterministic mock-upstream +responses. The subject includes the OpenCodex compatibility version, adapter, +inbound protocol, upstream protocol, surface, and relevant behavior +fingerprint. A real provider account is neither required nor permitted. + +A pass proves only the exercised OpenCodex translation. It says nothing about a +provider's current availability or a model's coding quality. + +### `live_route_compatibility` + +Question: does this exact configured route satisfy this versioned scenario now? + +The subject is an exact provider/model/effective-adapter/configuration route. +The run may contact only that route's configured upstream under the Lab +sandbox. A pass cannot be reused for a different route fingerprint. + +A pass proves only the exercised route behavior at the observation time. It +does not prove task effectiveness. + +### `task_effectiveness` + +Question: did this exact route produce a successful, deterministically verified +outcome for a versioned task class? + +Agent Fabric, not the Lab, owns real execution. The Lab receives a structured +outcome containing deterministic verifier results and sanitized artifact +references. Human ratings or LLM-judge output may be stored as advisory +annotations in a later phase, but cannot produce a canonical verdict. + +### Non-collapse rule + +The canonical projection key is: + +```text +(subjectId, evidenceLayer, suiteId, suiteVersion, suiteManifestDigest, + projectionSpecVersion) +``` + +There is no projection across all layers and no weighted universal score. +Callers may present multiple layer verdicts next to each other. A prerequisite +failure in one layer may make a later-layer run inapplicable, but it does not +rewrite evidence in the other layer. + +## 2. Immutable ledger contract + +The canonical JSONL ledger is a sequence of versioned events. The minimum +event kinds are: + +```text +observation +claim_snapshot +invalidation +``` + +An `observation` records one scenario attempt. A `claim_snapshot` captures the +local declared-capability inputs and source revisions needed to reproduce +`CLAIMED`: registry/config, cached catalog or native metadata, including the +adapter inference currently assembled by `src/routing/capability.ts`. An +`invalidation` identifies prior evidence that a later-discovered harness, +fixture, redaction, or integrity defect makes unusable. Invalidations append; +they never delete or edit prior lines. + +Each event has: + +```text +schemaVersion +eventId +eventKind +recordedAt +producer +producerVersion +``` + +An observation additionally has: + +```text +evidenceLayer +scenarioId +scenarioVersion +scenarioManifestDigest +suiteId +suiteVersion +suiteManifestDigest +fixtureDigests[] +subject +subjectId +startedAt +completedAt +executionMode fixture | live | fabric +attempt +limits +outcome pass | fail | blocked | inconclusive +assertions[] +failure? { class, code, retryable, attribution } +expectedFailure? +environment +artifactRefs[] +sourceRefs? +``` + +Rules: + +- Canonical JSON is RFC 8785 JSON Canonicalization Scheme (JCS), encoded as + UTF-8 with no BOM. +- `eventId` is lowercase + `sha256("ocx-lab:event:v1\0" || JCS(event without eventId))`. +- `subjectId` is lowercase + `sha256("ocx-lab:subject:v1\0" || JCS(subject))`. +- Scenario and suite manifests use the same JCS construction with domains + `ocx-lab:scenario-manifest:v1` and `ocx-lab:suite-manifest:v1`. Fixture + digests are lowercase + `sha256("ocx-lab:fixture:v1\0" || exact fixture bytes)`. A manifest's digest + field and storage path are excluded from its preimage. Domain text is UTF-8 + and terminated by one NUL byte. +- Every observation carries the exact scenario, suite and fixture digests it + executed. Version strings without matching digests are invalid evidence. +- Timestamps are UTC epoch milliseconds; duration alone is not sufficient. +- Assertions record expected value/shape, observed normalized value/shape, and + pass/fail. They never require raw prompts or unbounded bodies. +- `failure.attribution` records whether the failure is attributable to + OpenCodex, the exact route, the environment, or the harness. +- Artifact references contain digest, media type, byte count, redaction + policy, and local relative path; never an arbitrary filesystem path. +- `sourceRefs` may contain existing request IDs, route-decision IDs, or future + Fabric outcome IDs. It must not inline the referenced request or task. +- The shipped request-history explain surface already composes selection trace, + `PersistedUsageAttempt[]`, and final outcome. A later Lab consumer should link + that normalized composition instead of reading prompt-bearing + `responses-state.json` or treating generated Cursor task protobufs as durable + OpenCodex state. +- A structurally invalid or partially written line contributes no evidence and + is reported as ledger corruption. SQLite must be rebuildable from all valid + complete lines. + +The canonical scenario manifest, suite manifest and synthetic fixture bytes +referenced by an observation are retained as content-addressed +`scenario_manifest`, `suite_manifest`, and `fixture` artifacts. These contract +artifacts have indefinite retention while any non-invalidated observation +references them. A missing or digest-mismatched contract artifact makes that +observation unusable and yields `harness_failure`; projection code must never +substitute the current manifest for the historical one. + +The SQLite projection may cache derived verdict rows. Such rows must include +their `asOf`, projection spec, scenario/suite/fixture manifest digests, and +contributing event IDs. Deleting SQLite and replaying JSONL plus the referenced +content-addressed contract artifacts must reproduce them. + +## 3. Canonical verdict contract + +The closed verdict set is: + +```text +UNKNOWN +CLAIMED +PROBED +VERIFIED +DEGRADED +BLOCKED +UNSUPPORTED +``` + +Verdicts are projections, not mutable evidence fields. + +### `UNKNOWN` + +Produced when no current registry support claim and no current, valid, +attributable observation can classify the projection key. + +It can also result when all prior evidence became stale or was invalidated and +there is no current claim or blocker. Absence of a test is not +`UNSUPPORTED`. + +### `CLAIMED` + +Produced only by a current snapshotted positive local capability declaration +for the exact capability/subject, when no current executable evidence yields a +stronger state. The snapshot records whether the declaration came from explicit +provider config, Provider Registry, cached catalog, native metadata, or current +adapter inference rather than pretending every claim is registry-authored. + +A claim cannot produce `PROBED` or `VERIFIED`. A registry negative declaration +is shown as claim metadata but does not by itself prove `UNSUPPORTED`. + +### `PROBED` + +Produced when at least one required executable scenario completed with an +attributable pass, but the suite's versioned verification rule is not yet +satisfied. Examples are partial required-scenario coverage or a scenario whose +assertions establish reachability/shape but not full suite verification. + +Connectivity-only `/models` checks, registry discovery, doctor output, health +samples, and blocked attempts cannot produce `PROBED`. + +### `VERIFIED` + +Produced only when every requirement in the suite manifest's verification rule +is met by current, valid, attributable observations for the exact projection +key, and no newer current contradictory attributable failure remains +unresolved. + +The projection must expose: + +- exact contributing event IDs; +- suite/scenario versions and manifest digest; +- subject ID and full local subject; +- projection algorithm version and `asOf`; +- freshness calculation; +- any invalidations and contradictory events considered. + +An LLM judge, user assertion, registry declaration, successful model listing, +or mutable `verified=true` flag cannot produce `VERIFIED`. + +### `DEGRADED` + +Produced when executable evidence proves that the capability works only +partially, loses required semantics, violates a required assertion while a +usable subset remains, or requires a suite-declared workaround. + +Examples include malformed tool-call/result correlation, dropped reasoning +replay required by the suite, or incomplete stream semantics with otherwise +usable output. Environmental blockers cannot produce `DEGRADED`. + +### `BLOCKED` + +Produced when a current attempt cannot reach a compatibility assertion because +of an environmental or administrative precondition and no current +compatibility-attributable verdict should take precedence. + +Authentication, quota, region policy, local network failure, provider +transients, harness failure, or exhausted Lab budget can yield `BLOCKED`. The +projection retains those classes separately. + +A blocked retry does not erase a still-current `VERIFIED`, `PROBED`, +`DEGRADED`, or `UNSUPPORTED` verdict. Once that prior verdict is stale, +`BLOCKED` may become the current state until a conclusive run succeeds. + +### `UNSUPPORTED` + +Produced only by executable, suite-declared evidence that the exact capability +is unavailable by contract for this subject. The scenario must define an +unambiguous unsupported signal or negative-control assertion; a generic 4xx, +timeout, empty output, registry omission, or failed authentication is +insufficient. + +Expected rejection of a deliberately unsupported feature can prove +`UNSUPPORTED` when the rejection itself matches the deterministic contract. It +is not a failed harness run. + +### Projection precedence + +For current valid evidence at the same projection key: + +1. Satisfied full verification rule yields `VERIFIED`. +2. An unresolved attributable required-assertion failure yields `DEGRADED` or + `UNSUPPORTED` according to the scenario's failure rule. +3. Partial positive coverage yields `PROBED`. +4. A blocker yields `BLOCKED` only when 1-3 have no current result. +5. A current positive registry claim yields `CLAIMED`. +6. Otherwise the result is `UNKNOWN`. + +This is precedence, not a quality scale. In particular, `DEGRADED`, +`BLOCKED`, and `UNSUPPORTED` are not numeric values below `PROBED`. + +## 4. Transitions, contradiction and freshness + +Because verdicts are recomputed, a "transition" means that new events, time, or +version inputs change a projection. + +Allowed transitions: + +| From | May move to | Cause | +|---|---|---| +| `UNKNOWN` | any state | claim, observation, or blocker | +| `CLAIMED` | `UNKNOWN`, `PROBED`, `VERIFIED`, `DEGRADED`, `BLOCKED`, `UNSUPPORTED` | claim removal/staleness or executable evidence | +| `PROBED` | `UNKNOWN`, `CLAIMED`, `VERIFIED`, `DEGRADED`, `BLOCKED`, `UNSUPPORTED` | coverage, contradiction, staleness, invalidation | +| `VERIFIED` | `UNKNOWN`, `CLAIMED`, `PROBED`, `DEGRADED`, `BLOCKED`, `UNSUPPORTED` | partial remaining coverage after invalidation, contradiction, staleness, or changed inputs | +| `DEGRADED` | `UNKNOWN`, `CLAIMED`, `PROBED`, `VERIFIED`, `BLOCKED`, `UNSUPPORTED` | repair evidence, staleness, invalidation, or reclassification | +| `BLOCKED` | any state | blocker clears, prior evidence becomes current/stale, or claim changes | +| `UNSUPPORTED` | `UNKNOWN`, `CLAIMED`, `PROBED`, `VERIFIED`, `DEGRADED`, `BLOCKED` | route/version/config change, invalidation, or new evidence | + +Direct transitions not listed are forbidden; implementations must not invent a +state outside this set. + +Contradictory attributable evidence is never overwritten. The projection: + +1. filters by exact subject/layer/suite/scenario versions; +2. applies invalidation events; +3. applies freshness; +4. orders observations by completion time and deterministic event-ID tie-break; +5. applies the suite's contradiction rule; +6. emits the contributing and contradicting event IDs. + +The initial contradiction rule is conservative: a newer required-scenario +failure prevents `VERIFIED` until a newer pass of that scenario and all other +required coverage exists. A newer pass can restore `VERIFIED`; history remains. + +### Freshness + +Each scenario manifest declares its maximum evidence age. The suite manifest +may declare a stricter maximum, and a future Routing Profile may tighten it +again. Effective maximum age is the minimum of all finite scenario, suite and +profile values; `null` means no bound at that layer: + +- deterministic protocol evidence has no wall-clock expiry by default, but is + exact-match bound to scenario, suite, compatibility version, adapter and + behavior fingerprint; +- initial live-route manifests default to seven days; +- initial task-effectiveness manifests default to thirty days; +- a profile's maximum evidence age is an additional upper bound, never an + extension. + +Stale observations remain queryable but cannot support current +`PROBED`/`VERIFIED`/`DEGRADED`/`UNSUPPORTED`. The projection may display a +`lastKnownVerdict` separately. Its current verdict falls to `CLAIMED`, +`BLOCKED`, or `UNKNOWN` according to current inputs. + +### Version and configuration changes + +- Evidence matches an exact scenario and suite version in contract v1. No + implicit semver range reuse is allowed. +- A changed scenario assertion, fixture, requirement, or classification rule + requires a new scenario version and invalidates old evidence for the new + projection key. +- `opencodexCompatibilityVersion` is lowercase + `sha256("ocx-lab:compatibility-version:v1\0" || JCS(manifest))`. + The exact manifest object is: + + ```text + { + "schemaVersion": 1, + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "bunRuntimeVersion": , + "files": [ + { + "path": , + "sha256": + }, + ... + ] + } + ``` + + `files` contains every Git-index-tracked regular file under `src/`, plus + `package.json`, `bun.lock`, and `scripts/model-metadata.source.json`, sorted + by UTF-8 path bytes. Generation reads current working-tree bytes so a dirty + behavior change cannot reuse clean-tree evidence. A missing file, a tracked + symlink, a non-regular file, duplicate normalized path, invalid UTF-8 path, + or unreadable file makes the run `harness_failure`; untracked files are not + loaded by the compatibility harness. Release/package builds embed this + generated manifest so an installed runtime does not require Git. This + conservative whole-runtime input set may invalidate unrelated evidence, but + cannot falsely reuse evidence after a behavior change. The package marketing + version remains provenance only. +- A compatibility-version change starts a new subject projection. +- Any behavior-relevant configuration fingerprint change starts a new subject. +- Credential rotation alone does not change the subject. + +## 5. Failure-attribution contract + +Every non-pass observation uses exactly one primary class. Stable secondary +codes may add detail without changing these semantics. + +| Class | Meaning | Affects verdict? | Default action | +|---|---|---:|---| +| `protocol_failure` | OpenCodex or the exact route emitted, accepted, ordered, translated, or terminated protocol data incorrectly | Yes, in the observation's layer | Conclusive; reverify after code/config/version change | +| `capability_failure` | The exact route cannot satisfy a capability assertion that it was expected to support | Yes | Conclusive when the scenario rules out an unsupported contract; otherwise retry once then `inconclusive` | +| `behavioral_failure` | A task-effectiveness deterministic verifier failed although protocol/capability prerequisites completed | Yes, task layer only | Conclusive for that task scenario | +| `authentication_blocked` | Missing, expired, rejected, or insufficient credentials prevented the assertion | No | `BLOCKED`; reauthenticate and retry | +| `quota_blocked` | Rate, credit, token, concurrency, or account quota prevented the assertion | No | `BLOCKED`; retry after reset/backoff | +| `region_blocked` | Region/tenant policy prevented execution | No | `BLOCKED`; retry only when route context changes | +| `network_failure` | DNS, TLS establishment, connect, local proxy, or transport reachability failed without provider response evidence | No | `BLOCKED`; repair environment and retry | +| `provider_transient` | Upstream returned a recognized transient/overload failure or interrupted a previously valid service path | No by default | `BLOCKED`; bounded retry/reverification | +| `timeout` | A versioned scenario deadline expired; secondary code distinguishes connect, first-byte, inactivity, or total budget | No by default | `BLOCKED`; bounded retry; reclassify only with deterministic protocol evidence | +| `harness_failure` | Runner, fixture, mock, sandbox, assertion engine, or artifact writer failed | No | Invalidate affected evidence and fix harness | +| `budget_exhausted` | Lab request/token/tool/byte/time budget ended the run before its assertion | No | `BLOCKED`; revise scenario limits/version or retry | +| `inconclusive` | Observations conflict or lack enough information for another class | No | No promotion/degradation; investigate/reverify | + +Safety rules: + +- Expired credentials never imply broken tool support. +- Quota exhaustion never implies model incompatibility. +- Local DNS/TLS/connect failure never degrades provider capability. +- A generic timeout never proves missing terminal semantics. A deterministic + mock stream that closes without its required terminal event is + `protocol_failure`; a live body that simply stalls is `timeout`. +- Malformed tool-call semantics, broken tool-result correlation, or lost + required event ordering may legitimately produce `protocol_failure` and + `DEGRADED`. +- `provider_transient` may be promoted to a compatibility-affecting class only + by a scenario-specific deterministic rule and a new observation; projection + code must not infer promotion from retry count. +- An expected failure is first-class scenario data. When the observed + rejection exactly matches a declared unsupported assertion, it can produce + `UNSUPPORTED`. Any other expected failure remains a pass/fail of the + assertion, not a blanket suppression. + +## 6. Canonical route subject + +Evidence is never keyed by model name alone. `RouteSubjectV1` contains: + +```text +subjectSchemaVersion +providerId +providerInstanceFingerprint +clientModelId +upstreamModelId +effectiveAdapter +inboundProtocol +upstreamProtocol +surface +opencodexCompatibilityVersion +behaviorFingerprint +endpointFingerprint +dependencies[] +``` + +Semantics: + +- `providerId` is the built-in registry ID or `custom`; it is not a display + label. +- `providerInstanceFingerprint` is a locally salted HMAC over the configured + provider identity, allowing two instances of one preset to differ without + leaking a user-selected name. All local opaque fingerprints use lowercase + HMAC-SHA-256 over + `UTF8("ocx-lab:local-fingerprint:v1\0" + fieldName + "\0") || JCS(value)` + with the installation salt as key. +- `clientModelId` is the selected canonical route model; `upstreamModelId` is + the effective wire model after namespace, virtual-model, combo and suffix + resolution. +- `effectiveAdapter` reflects model-specific wire defaults/overrides and wire + pins, not merely the provider-wide configured adapter. +- Protocol values distinguish OpenAI Responses, OpenAI Chat Completions, + Anthropic Messages, and provider-specific wires. +- `surface` distinguishes behaviorally different ingress/transport paths such + as Responses HTTP, Responses WebSocket, Chat HTTP/SSE, and Anthropic + Messages HTTP/SSE. +- The package/build version remains observation provenance in + `producerVersion`; only `opencodexCompatibilityVersion` participates in the + subject so an unrelated release does not discard valid conformance evidence. +- `endpointFingerprint` is a locally salted HMAC of the normalized destination + scheme/host/port/base path. Raw URLs, userinfo, query strings, and fragments + are not evidence fields. +- `dependencies` is an ordered list of flat `RouteDependencyV1` records for + behaviorally invoked sidecars. Each record contains role, provider ID, + provider-instance fingerprint, client/upstream model IDs, effective adapter, + upstream protocol, endpoint fingerprint, and behavior fingerprint. It cannot + nest. Records sort by role, provider ID, upstream model ID, then endpoint + fingerprint. An empty list is canonical when no sidecar is invoked. + +### Behavior fingerprint allowlist + +The fingerprint is SHA-256 over canonical JSON containing only effective, +behavior-changing values applicable to the selected model/surface: + +- adapter/wire resolution and `responsesPath`; +- auth mode and auth transport, but no credential/account identity; +- stateful/stateless Responses behavior, upstream streaming mode, service-tier + support, snapshot and item-ID repair; +- context/input/output limits and input modalities; +- reasoning capability, effort/default/mapping/wire/summary/replay/split/ + toggle/budget behavior; +- tool-choice restrictions, parallel-tool support, hosted-tool preference, + freeform/custom-tool handling, built-in-name escaping; +- prompt-cache forwarding, Anthropic EOF policy, model suffix handling; +- Google mode and opaque project/location fingerprints where applicable; +- OpenRouter routing preferences where applicable; +- actual Bun runtime version and platform/architecture whenever the selected + stream path or adapter has platform-sensitive behavior; +- effective vision and web-search sidecar enablement, backend, model, + reasoning, per-turn limits, timeout/stall limits, and the matching flat + dependency subject when that sidecar can execute; +- MCP schema/result/tool count bounds, with all Lab execution facilities forced + to the sandbox settings in the security contract; +- effective global stream mode, fast/service-tier behavior, and effort caps + when they can alter the scenario; +- a digest of non-credential custom header behavior. + +Values that are inapplicable to the selected model/surface are omitted rather +than copied wholesale. Canonical JSON sorts keys, normalizes absent/default +values to their effective value, and sorts set-like arrays while preserving +order-sensitive arrays. + +The following never participate: + +- API keys, OAuth/access/refresh tokens, cookies, authorization headers; +- account IDs, emails, labels, aliases, quota balances or plan names; +- raw custom/private header names or values; +- prompts, messages, tool results, repository paths or contents; +- timestamps, transient health, latency, cost, quota or retry state. + +Credential headers are excluded. Non-credential custom headers contribute only +through a locally salted HMAC over normalized names/values, so a behavior +change alters identity without disclosing the header. Project/location and +custom endpoint values use the same local opaque treatment. + +Public export replaces all local fingerprints with export-scoped opaque IDs +and redacts custom model IDs unless the export policy explicitly classifies +them as public. + +## 7. Task-effectiveness ingress + +A future Fabric observation must provide, at minimum: + +```text +producerSchemaVersion +outcomeId +taskClassId +taskClassVersion +subject +startedAt +completedAt +resourceLimits +result success | failure | blocked | inconclusive +verifiers[] { id, version, result, normalizedMetrics? } +artifactRefs[] +``` + +The Lab rejects an outcome if the subject cannot be reconstructed, a verifier +is nondeterministic for a canonical assertion, or an artifact violates the +security contract. Free-form narrative may be retained only as bounded, +sanitized advisory metadata and never determines the verdict. + +## 8. Consumer boundaries + +- Provider Registry supplies claims; Lab does not rewrite them. +- Request history supplies route/outcome references; Lab does not copy its + ledger. +- Selection and execution stay separate: `RouteDecisionTraceV1` records the + pre-dispatch choice, `attempts[]` records physical execution/fallback, and + final outcome is joined at read time. +- Routing Profiles express user requirements; Lab does not evaluate policy. +- Router Intelligence reads projections; Lab does not rank candidates. +- Route decision traces explain compatibility exclusions/penalties in the + existing trace; Lab does not create a parallel route explanation. +- Agent Fabric executes tasks; Lab does not run arbitrary repository work. diff --git a/devlog/_plan/260807_compatibility_lab/020_scenario_contract_and_catalogue.md b/devlog/_plan/260807_compatibility_lab/020_scenario_contract_and_catalogue.md new file mode 100644 index 000000000..fa0e610ff --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/020_scenario_contract_and_catalogue.md @@ -0,0 +1,394 @@ +# CL-00 scenario contract and initial catalogue + +This document freezes the scenario schema and the initial IDs. CL-01 implements +the `*.protocol.*` scenarios only. Entries marked live or Fabric-reserved define +future semantics and are not authorization to execute them. + +The normative V1 selector/operator semantics, immutable fixture anchors, +expanded defaults and complete protocol scenario/suite manifest records are in +[the protocol V1 manifest authority](./021_protocol_v1_manifest_authority.md); +its canonical fixture vectors and literal expectations are in +[`022_protocol_v1_cases.json`](./022_protocol_v1_cases.json). + +## 1. Versioned scenario model + +A `CompatibilityScenarioV1` has: + +```text +schemaVersion 1 +id stable lowercase dotted ID +version exact semver +suite { id, version } +evidenceLayer protocol_conformance | + live_route_compatibility | + task_effectiveness +capability stable capability ID +verificationRole required | supplemental | negative_control +requirements +fixtures +executionLimits +assertions[] +failureRules[] +artifactPolicy +freshness +``` + +### Identity and versioning + +- `id` names semantics and does not contain a provider, model, or version. +- `version` is exact-match in contract v1. +- Any assertion, fixture, limit that affects expected behavior, failure rule, + artifact exposure, or requirement change increments the scenario version. +- Editorial description changes do not require a version change. +- A suite manifest has its own exact version and lists scenario IDs, versions, + roles, and its verification rule. +- Scenario and suite manifests are RFC 8785 canonical JSON with the + domain-separated digest defined by the evidence contract. + +### Requirements + +Requirements are declarative and may include: + +```text +inboundProtocols[] +upstreamProtocols[] +surfaces[] +requiredClaims[] +requiredHarnessFeatures[] +platforms[] +routePreconditions[] +``` + +An unmet deterministic fixture requirement is `harness_failure`. An unmet live +route precondition is either inapplicable or a typed blocker; it is never +silently counted as a capability failure. + +### Fixtures + +Fixture references include ID, content digest, media type, generator version +where generated, and role (`client_request`, `upstream_response`, +`adapter_vector`, `synthetic_tool`, `synthetic_image`, or `task_fixture`). A +fixture never embeds credentials, user data, or an external mutable URL. + +Protocol fixtures run against a deterministic mock upstream. Live scenarios use +only Lab-owned synthetic requests and inert tools. Fabric scenarios refer to a +versioned synthetic task class; they do not place a repository in the Lab +ledger. + +### Execution limits + +Every scenario states: + +```text +totalTimeoutMs +connectTimeoutMs? +firstByteTimeoutMs? +inactivityTimeoutMs? +maxRequests +maxInputBytes +maxOutputBytes +maxOutputTokens? +maxToolCalls +maxArtifactBytes +``` + +Absent limits are invalid. Limits may be stricter than Lab-wide ceilings but +not wider without a scenario version change and security review. Expiry of a +limit classifies as `timeout` or `budget_exhausted` according to the failed +limit; it does not imply incompatibility. + +### Deterministic assertion DSL + +Canonical assertions use a closed set of observable operators: + +```text +http_status_equals +header_present +header_absent +header_value_equals +json_schema_matches +json_path_equals +json_path_present +json_path_absent +sse_field_equals +sse_event_sequence +sse_event_count +terminal_signal_equals +id_matches +id_stable_across_events +id_correlates +tool_call_equals +tool_result_correlates +fixture_request_matches +normalized_text_equals +byte_limit_observed +process_exit_equals +verifier_result_equals +``` + +Each assertion has an ID, operator, selector, expected value, required flag, +and redaction-safe observed summary. Implementations must exhaustively reject +unknown operators. Protocol V1 permits no arbitrary regular expressions; ID +grammars and all operator type/missing-value behavior are closed in the +manifest authority. + +Core verdicts cannot use an LLM judge, free-form human interpretation, or a +snapshot that contains unstable timestamps/IDs without normalization. + +### Failure rules + +Rules are ordered and explicit: + +```text +match assertion IDs, normalized status/error/event/timeout +classification canonical failure class +secondaryCode +verdictEffect none | degraded | unsupported +retry never | bounded | after_precondition_change +expected boolean +``` + +The first exact rule wins. If no rule establishes a compatibility-attributable +class, the attempt is `inconclusive`. Generic HTTP 4xx/5xx rules may classify a +blocker or transient but cannot prove `UNSUPPORTED`. + +### Artifact policy + +The policy is deny-by-default and names allowed normalized artifacts: + +```text +assertion_report +sanitized_request_shape +sanitized_response_shape +normalized_event_trace +sanitized_error +verifier_summary +``` + +It states per-artifact and aggregate byte limits, retention class, local/public +visibility, and redaction profile. Raw credentials, prompts, hidden reasoning, +full task repositories, arbitrary headers, and arbitrary response bodies are +not valid artifact kinds. + +Each scenario declares `freshness.maxAgeMs`; its suite may declare a stricter +bound, and a future profile may tighten it again. Effective maximum age is the +minimum finite bound, with `null` meaning unbounded at that layer. + +## 2. Suite projection rules + +For each exact suite manifest: + +- `VERIFIED`: all `required` scenarios applicable to the subject have current + passes and every negative control observed its required rejection. +- `PROBED`: at least one required scenario passed, with no current + compatibility-attributable required-scenario failure, but coverage is + incomplete. +- `DEGRADED`: a failure rule on an applicable required scenario yields + `degraded`. +- `UNSUPPORTED`: a required scenario's deterministic unsupported rule or + negative control proves the capability unavailable. +- `BLOCKED`: only blockers exist and no current attributable verdict takes + precedence. +- `CLAIMED`/`UNKNOWN`: follow the evidence contract. + +Supplemental scenarios never block `VERIFIED` unless a new suite version makes +them required. + +## 3. Initial suite catalogue + +All initial scenario versions and suite versions are `1.0.0`. + +For protocol V1, the literal fixtures/assertions in `022` are the complete +verification boundary. Descriptions below summarize those exact vectors; they +do not silently incorporate every historical incident mapped to the same +scenario ID. An incident absent from `022` is candidate coverage for a reviewed +scenario/suite version amendment and cannot be claimed by a V1 `VERIFIED` +verdict. + +### `responses-core` + +Purpose: preserve the OpenAI Responses request, output-item lifecycle, stream +framing, IDs, terminal state, and JSON/SSE equivalence. + +Capability: `protocol.responses.core`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `responses-core.protocol.request-shape` | Deterministic; CL-01 | Mock receives the exact model, first user text and zero temperature | +| `responses-core.protocol.sse-framing` | Deterministic; CL-01 | Spaced and unspaced data fields, a non-record `null` frame, data-only event inference, exact text and completion terminal | +| `responses-core.protocol.item-lifecycle` | Deterministic; CL-01 | One added/done/completed lifecycle with stable valid message ID | +| `responses-core.protocol.terminal-state` | Deterministic; CL-01 | One explicit failed terminal is preserved exactly | +| `responses-core.protocol.json-sse-equivalence` | Deterministic; CL-01 | One normalized JSON/SSE pair has equal text and completion terminal | +| `responses-core.live.basic-turn` | Live-reserved | 2xx, bounded output, valid lifecycle and terminal state from exact route | + +Unsupported means a route deterministically rejects the Responses surface with +a suite-recognized unsupported signal. Semantic loss, invalid IDs, malformed +event order, or missing terminal state is degraded. Auth/quota/region/network, +transient upstream errors, and body stalls are blocked. + +### `chat-core` + +Purpose: preserve OpenAI Chat Completions request/response semantics for JSON +and streaming routes. + +Capability: `protocol.chat.core`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `chat-core.protocol.request-mapping` | Deterministic; CL-01 | System/developer/user order and JSON-object response format match the fixture | +| `chat-core.protocol.nonstream-envelope` | Deterministic; CL-01 | One valid choice/message/finish/usage envelope yields exact text and terminal | +| `chat-core.protocol.stream-assembly` | Deterministic; CL-01 | Fragmented/interleaved deltas assemble in order and emit one finish | +| `chat-core.protocol.stream-terminal` | Deterministic; CL-01 | One stop finish plus `[DONE]` yields exact text and one terminal | +| `chat-core.live.basic-turn` | Live-reserved | Exact route returns bounded text and a valid finish contract | + +Unsupported is a deterministic surface rejection. Incorrect role mapping, +malformed choices, lost stream fragments, or invalid finish semantics is +degraded. Environmental and transient failures are blocked. + +### `anthropic-core` + +Purpose: preserve Anthropic Messages roles/content blocks, tool/thinking block +ordering, stop reasons, usage, and SSE lifecycle. + +Capability: `protocol.anthropic.messages.core`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `anthropic-core.protocol.request-mapping` | Deterministic; CL-01 | Model, system instruction and first user text map exactly | +| `anthropic-core.protocol.content-sequence` | Deterministic; CL-01 | `message_start`, monotonic content blocks/deltas/stops, `message_delta`, `message_stop` | +| `anthropic-core.protocol.tool-round-trip` | Deterministic; CL-01 | One `tool_use`/`tool_result` pair preserves its ID correlation and result text | +| `anthropic-core.protocol.terminal-errors` | Deterministic; CL-01 | One explicit Responses failure maps to the Anthropic error/failed terminal | +| `anthropic-core.live.basic-turn` | Live-reserved | Exact route returns a valid bounded Messages lifecycle and terminal | + +Unsupported is a recognized Messages-surface rejection. Wrong block ordering, +lost tool correlation, invalid stop reason, or clean EOF accepted without the +suite's terminal contract is degraded. Authentication, quota, region, network, +transient failure, and silence timeout are blocked. + +### `tools-core` + +Purpose: prove deterministic function/custom tool declaration, call assembly, +parallel correlation, and result continuation. + +Capability: `tools.round_trip`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `tools-core.protocol.function-round-trip` | Deterministic; CL-01 | One function call preserves ID/name/parsed arguments and its continuation result correlates | +| `tools-core.protocol.custom-freeform-round-trip` | Deterministic; CL-01 | One `apply_patch` call preserves exact freeform input and its continuation result correlates | +| `tools-core.protocol.parallel-correlation` | Deterministic; CL-01 | Two interleaved calls assemble once in first-seen order without overlap | +| `tools-core.protocol.result-content` | Deterministic; CL-01 | One result preserves exact text and data-image parts | +| `tools-core.protocol.choice-and-allowed-set` | Deterministic; CL-01 | One required single-tool allowed set narrows exactly without widening | +| `tools-core.live.function-round-trip` | Live-reserved | Inert deterministic function is called once with schema-valid args and static result is continued | +| `tools-core.live.custom-freeform-round-trip` | Live-reserved | Route emits exact custom/freeform call and accepts static result continuation | + +An explicit route contract that rejects a tool kind can prove unsupported. +Malformed arguments, dangling IDs, widened choice, dropped calls/results, or +incorrect parallel assembly is degraded. A model choosing not to call an +`auto` tool is inconclusive; required tool choice is used for conclusive live +coverage. Environmental failures are blocked. + +### `codex-core` + +Purpose: establish the minimum end-to-end semantics required to advertise a +route as usable by Codex. A basic Responses text request is insufficient. + +Capability: `client.codex.core`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `codex-core.protocol.streaming-turn` | Deterministic; CL-01 | One Chat-backed stream yields exact text, final-answer phase and one completed terminal | +| `codex-core.protocol.apply-patch-turn` | Deterministic; CL-01 | One custom `apply_patch` call preserves exact patch text and its result ID correlates | +| `codex-core.protocol.tool-continuation` | Deterministic; CL-01 | One function result follows and correlates with its prior call | +| `codex-core.protocol.previous-response-replay` | Deterministic; CL-01 | One local expansion preserves stored input/output/new-input order and strips `previous_response_id` | +| `codex-core.protocol.structured-output` | Deterministic; CL-01 | One JSON-schema request maps to the exact Chat `response_format` | +| `codex-core.protocol.compaction-and-special-items` | Deterministic; CL-01 | Compaction, local shell, tool search and hosted-tool items are normalized without leaking opaque raw data | +| `codex-core.live.tool-turn` | Live-reserved | Valid stream plus required inert tool call/result continuation and terminal | +| `codex-core.live.custom-tool-turn` | Live-reserved | Required custom/freeform call/result continuation and valid terminal | + +The `codex-core` manifest requires all six protocol scenarios for conformance +verification. Future live verification requires both live scenarios plus +current `responses-core.live.basic-turn`. Text-only success is at most partial +coverage, never `codex-core: VERIFIED`. + +An exact route may be unsupported when it deterministically lacks a mandatory +Codex surface or tool kind. Lossy lifecycle, call/result correlation, +continuation, or special-item behavior is degraded. Environmental failures are +blocked. + +### `vision-core` + +Purpose: preserve declared image input and tool-result image behavior and prove +exact-route image understanding without user media. + +Capability: `modalities.image.input`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `vision-core.protocol.input-image` | Deterministic; CL-01 | One data-URL image preserves detail and text/image ordering | +| `vision-core.protocol.tool-result-image` | Deterministic; CL-01 | Image content in function/tool result remains structured and correlated | +| `vision-core.protocol.modality-gate` | Deterministic negative control; CL-01 | A text-only/no-sidecar synthetic vector produces the typed unsupported path without silent image drop | +| `vision-core.live.synthetic-ocr` | Live-reserved | Lab-generated image nonce is returned in an exact JSON schema | + +A deterministic declared no-image contract may prove unsupported. Dropping, +textifying without a declared sidecar, corrupting, or misordering image content +is degraded. Failure of the optional sidecar route is attributed to that exact +subject. Auth/quota/network/transient failures are blocked. + +### `reasoning-core` + +Purpose: preserve supported reasoning controls, summaries, signatures and +replay while preventing provider-private reasoning material from crossing an +incompatible boundary. + +Capability: `reasoning.round_trip`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `reasoning-core.protocol.effort-mapping` | Deterministic; CL-01 | Effective effort maps to the declared wire form and unsupported parameters are omitted | +| `reasoning-core.protocol.summary-stream` | Deterministic; CL-01 | One summary-part/delta/completed sequence preserves ordering and reasoning ID | +| `reasoning-core.protocol.replay` | Deterministic; CL-01 | One synthetic plaintext/signature replay reaches the second turn exactly | +| `reasoning-core.protocol.private-content-isolation` | Deterministic; CL-01 | One provider-private value is absent from an incompatible upstream and client response | +| `reasoning-core.live.replay` | Live-reserved | Synthetic two-turn route accepts its declared replay form and completes | + +An explicit no-reasoning route can prove unsupported. Rejected or corrupted +declared replay, lost required signatures, or private content sent across an +incompatible provider boundary is degraded (and the latter is also a security +finding). Environmental failures are blocked. + +### `mcp-core` + +Purpose: preserve MCP namespace, schema, tool/resource invocation and result +contracts through supported adapters without touching user MCP servers. + +Capability: `tools.mcp.core`. + +| Scenario ID | Layer/applicability | Required observable assertions | +|---|---|---| +| `mcp-core.protocol.namespace-mapping` | Deterministic; CL-01 | One namespace/name pair flattens and reverses exactly | +| `mcp-core.protocol.schema-and-bounds` | Deterministic; CL-01 | Tool schemas encode correctly; exact configured bounds admit and one-byte-over rejects atomically | +| `mcp-core.protocol.call-result` | Deterministic; CL-01 | Lab stub receives one exact call and returns one successful text result | +| `mcp-core.protocol.resource-round-trip` | Deterministic; CL-01 | One list/read resource success shape preserves URI, name and text | +| `mcp-core.live.synthetic-tool` | Live-reserved | Lab-owned loopback pure-function MCP tool is advertised, called and correlated | + +Only a Lab-owned in-memory or loopback fixture is allowed. A route that +deterministically cannot expose MCP may be unsupported. Namespace loss, schema +corruption, partial bound commits, or result miscorrelation is degraded. +User-server unavailability is never tested; environmental failures are blocked. + +## 4. CL-01 implementation boundary + +CL-01 may implement the scenario registry, deterministic mock-upstream harness, +closed assertion DSL, and only the `protocol_conformance` manifests frozen in +`021_protocol_v1_manifest_authority.md`. It must not: + +- contact a real provider; +- write the planned production evidence ledger or SQLite projection; +- add profile/routing controls; +- execute a user tool, shell, filesystem, repository, MCP server, or external + network action; +- implement live/Fabric scenarios merely because their IDs are reserved here. + +If CL-01 discovers that an observable assertion cannot be implemented without +new semantics, it must amend this contract in a reviewed change rather than +quietly inventing behavior. diff --git a/devlog/_plan/260807_compatibility_lab/021_protocol_v1_manifest_authority.md b/devlog/_plan/260807_compatibility_lab/021_protocol_v1_manifest_authority.md new file mode 100644 index 000000000..5c5035176 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/021_protocol_v1_manifest_authority.md @@ -0,0 +1,342 @@ +# CL-00 protocol V1 manifest authority + +This document closes the executable semantics for the initial +`protocol_conformance` scenarios. It is normative for CL-01 and does not +implement a runner. + +The machine-readable source of truth is +[`022_protocol_v1_cases.json`](./022_protocol_v1_cases.json). It contains 35 +provider-independent canonical fixture vectors, literal expected values, +row-specific requirements, exact roles/media types, execution limits, artifact +policy, failure rules, and domain-separated fixture digests. Historical tests +in the [incident corpus](./030_incident_corpus.md) are provenance and coverage +guidance only; they are not executable manifest semantics. + +## 1. Exact manifest expansion + +For each entry in `cases`, CL-01 constructs `CompatibilityScenarioV1` in this +field order before RFC 8785 canonicalization: + +```text +schemaVersion source.schemaVersion +id case.id +version manifestDefaults.version +suite { id: case.suite, + version: manifestDefaults.suiteVersion } +evidenceLayer manifestDefaults.evidenceLayer +capability case.capability +verificationRole case.verificationRole when present, + otherwise manifestDefaults.verificationRole +requirements case.requirements +fixtures when case.initiatingRequest is present: + [fixtureRef(case.initiatingRequest), + fixtureRef(case.fixture)] + otherwise: [fixtureRef(case.fixture)] +executionLimits manifestDefaults.executionLimits +assertions case.assertions +failureRules failureRuleSets[ + manifestDefaults.failureRuleSet] +artifactPolicy manifestDefaults.artifactPolicy +freshness manifestDefaults.freshness +``` + +`fixtureRef(x)` is exactly: + +```text +{ + id: x.id, + role: x.role, + mediaType: x.mediaType, + digest: x.digest, + byteLength: UTF8(x.bytesUtf8).byteLength +} +``` + +JSON object field order has no digest effect, but the field set above is +closed. Unknown fields reject registration. Arrays preserve source order. +Empty arrays remain present. No default may be read from runtime code. + +The scenario digest is: + +```text +sha256( + UTF8("ocx-lab:scenario-manifest:v1\0") || + UTF8(JCS(expanded scenario)) +) +``` + +The exact fixture bytes are UTF-8 encoding of each `bytesUtf8` field; every +published `fixture` and `initiatingRequest` digest is: + +```text +sha256(UTF8("ocx-lab:fixture:v1\0") || fixture bytes) +``` + +Registration recomputes all digests and rejects mismatch. Every `bytesUtf8` is +retained as a content-addressed fixture artifact, not duplicated into the +expanded manifest. An `upstream_response` case without an initiating +`client_request` rejects registration. + +## 2. Exact suite manifests + +All suite versions are `1.0.0`. Every listed scenario has role `required` +except `vision-core.protocol.modality-gate`, whose role is `negative_control`. +No supplemental protocol V1 scenario exists. Order is the order in the case +authority: + +| Suite | Capability | Required member suffixes | +|---|---|---| +| `responses-core` | `protocol.responses.core` | `request-shape`, `sse-framing`, `item-lifecycle`, `terminal-state`, `json-sse-equivalence` | +| `chat-core` | `protocol.chat.core` | `request-mapping`, `nonstream-envelope`, `stream-assembly`, `stream-terminal` | +| `anthropic-core` | `protocol.anthropic.messages.core` | `request-mapping`, `content-sequence`, `tool-round-trip`, `terminal-errors` | +| `tools-core` | `tools.round_trip` | `function-round-trip`, `custom-freeform-round-trip`, `parallel-correlation`, `result-content`, `choice-and-allowed-set` | +| `codex-core` | `client.codex.core` | `streaming-turn`, `apply-patch-turn`, `tool-continuation`, `previous-response-replay`, `structured-output`, `compaction-and-special-items` | +| `vision-core` | `modalities.image.input` | `input-image`, `tool-result-image`, `modality-gate` | +| `reasoning-core` | `reasoning.round_trip` | `effort-mapping`, `summary-stream`, `replay`, `private-content-isolation` | +| `mcp-core` | `tools.mcp.core` | `namespace-mapping`, `schema-and-bounds`, `call-result`, `resource-round-trip` | + +For each row, the expanded suite manifest is: + +```text +schemaVersion 1 +id table suite +version 1.0.0 +evidenceLayer protocol_conformance +capability table capability +assertionDslVersion 1.0.0 +evidenceSchemaVersion 1.0.0 +freshness { maxAgeMs: null } +contradictionRule newest-required-observation-v1 +scenarios [{ + id: full case ID, + version: 1.0.0, + role: expanded scenario verificationRole, + manifestDigest: expanded scenario digest + }, ...] +verificationRule all-applicable-required-pass-v1 +``` + +Unknown fields reject registration. The suite digest uses +`ocx-lab:suite-manifest:v1` plus JCS exactly as defined by the evidence +contract. `VERIFIED` requires a current pass for every applicable member and +the exact suite/scenario/fixture digests above. + +## 3. Observation selector model + +Assertions use absolute RFC 6901 JSON Pointers into this closed normalized +observation: + +```text +{ + "client": { + "request": { "status": 0, "headers": {}, "json": null, "rawBytes": 0 }, + "response": { + "status": 0, + "headers": {}, + "json": null, + "events": [], + "terminal": null, + "normalizedText": "" + } + }, + "upstream": { + "requests": [ + { "status": 0, "headers": {}, "json": null, "rawBytes": 0 } + ], + "responses": [] + }, + "process": { "exitCode": null }, + "verifiers": {} +} +``` + +Header names are lowercase. JSON pointers use `~0` and `~1` escaping. Array +indexes are decimal with no leading zero except `0`; `-` is forbidden. +Wildcard, filter, recursive descent, script expression, URI, and filesystem +selectors do not exist in V1. + +Unless the operator is `json_path_absent`, a missing selector fails with +`selector_missing`. Unless the operator checks presence/absence, a wrong JSON +type fails with `selector_type_mismatch`. Both are required-assertion failures, +not harness failures. + +Objects compare by JCS bytes. Arrays are order-sensitive. Strings compare +without trimming or Unicode normalization. Numbers use JCS representation. +`null`, missing, empty string, empty array, and empty object are distinct. + +## 4. Fixture roles and execution + +Closed V1 fixture roles are: + +- `client_request`: inject exact bytes at the named inbound protocol surface; +- `upstream_response`: return exact bytes from the loopback mock; +- `adapter_vector`: decode the fixture JSON and feed its documented fields to + the selected adapter boundary without network access; +- `synthetic_tool`: decode the fixture JSON into the in-memory inert tool/MCP + stub. It never executes model arguments. + +Closed media types are `application/json`, `text/event-stream`, +`application/vnd.opencodex.adapter-vector+json`, and +`application/vnd.opencodex.mcp-stub+json`. + +Each case's exact `requirements` selects the adapter/surface and harness +features. `adapter_vector` keys are scenario-specific closed input fields +defined by the literal vector and scenario assertions; unknown keys reject the +fixture. CL-01 must encode those fields as a discriminated union keyed by the +scenario ID, not a generic callback or dynamic module. + +The MCP cases use only `in_memory_mcp_stub`. No case authorizes stdio, a child +process, user MCP configuration, filesystem access, or a user tool. + +## 5. SSE normalization + +The harness retains exact fixture bytes and normalizes only for assertions: + +1. UTF-8 must decode without replacement. A BOM is allowed only at byte zero + and is removed. +2. CRLF and CR become LF. +3. An empty line terminates a frame. Comment lines beginning `:` are ignored. +4. The first `:` separates field and value. No colon means an empty value. + Exactly one optional leading U+0020 after `:` is removed; no other + whitespace is trimmed. +5. Repeated `data` fields join with LF. The last `event` field wins. +6. `[DONE]` is a sentinel only for Chat surfaces. +7. When `event` is absent and parsed `data` is an object with string `type`, + Responses/Anthropic normalization infers that `type`. Explicit event wins. + Parsed `null`, scalar, array, or empty data is padding and emits no event. + Syntactically malformed nonempty JSON is terminal. +8. Arrival order is preserved; events are never sorted or deduplicated. + +Each normalized event is: + +```text +{ "event": string, "data": JSON value, "ordinal": integer } +``` + +## 6. Assertion operators + +- `http_status_equals`: selected integer equals expected integer. +- `header_present` / `header_absent`: selected lowercase header key exists/does + not exist. +- `header_value_equals`: selected normalized header string equals expected. +- `json_schema_matches`: selected value validates against embedded JSON Schema + draft 2020-12. Only local `$defs`/`$ref` are allowed; coercion, defaults, + custom formats, and network resolution are forbidden. +- `json_path_equals`: selected value equals literal expected under JCS rules. +- `json_path_present` / `json_path_absent`: pointer succeeds/fails; a present + `null` is present. Expected must be literal `true`. +- `sse_field_equals`: selected normalized field equals expected string. +- `sse_event_sequence`: exact event-name array; no subsequence or extras. +- `sse_event_count`: expected is `{event,count}` and exact count is required. +- `terminal_signal_equals`: expected is `completed`, `failed`, `incomplete`, + `done`, `message_stop`, `eof_tolerated`, or `none`. Exactly one terminal is + required unless expected is `none`. +- `id_matches`: expected is a closed grammar: + - `responses_message`: `msg_` plus 1..128 ASCII alphanumeric/underscore/dash; + - `responses_reasoning`: `rs_` plus 1..128 of that set; + - `responses_call`: `call_` plus 1..128 of that set; + - `nonempty_128`: 1..128 printable non-whitespace ASCII characters. + Arbitrary regular expressions are forbidden. +- `id_stable_across_events`: expected is an ordered pointer list. Every + resolved string is byte-equal. +- `id_correlates`: expected is exactly two pointers resolving to byte-equal + strings. +- `tool_call_equals`: selected normalized call equals + `{id,name,arguments,kind,ordinal}`. Function arguments are parsed JSON; + custom/freeform arguments are exact strings. +- `tool_result_correlates`: expected is `{call,result}` pointers. IDs match, + result follows call, and no intervening call reuses the ID. +- `fixture_request_matches`: method, normalized path, allowlisted headers and + JSON body equal the literal fixture expectation. +- `normalized_text_equals`: selected string equals expected exactly. +- `byte_limit_observed`: selected nonnegative integer is `<=` expected. +- `process_exit_equals`: selected integer equals expected. +- `verifier_result_equals`: selected value `pass|fail|blocked|inconclusive` + equals expected. + +Unknown operators, selectors, expected shapes, fixture roles, media types, or +requirements reject registration. + +## 7. Closed verifier derivations + +`/verifiers` is populated only by these pure V1 functions. They may read the +current case's decoded synthetic fixture and normalized observation, but no +clock, random source, network, filesystem, environment, runtime callback, or +model output outside that observation. + +- `json_sse_equivalence`: build the JSON projection + `{text,terminal}` where `text` concatenates, in order, every + `output[].content[]` `output_text.text`, and `terminal` is top-level + `status`. Build the SSE projection where `text` concatenates every + `response.output_text.delta.data.delta`, and `terminal` is the normalized + terminal. Return `pass` iff the two JCS objects are equal, else `fail`. +- `nonoverlap_order`: read normalized `tool_call` events in ordinal order. + Return their IDs only when every ID occurs once, ordinals are contiguous + from zero, and each event contains its complete arguments. Otherwise return + an empty array. +- `call_result_order`: over the normalized two-turn input, return `pass` iff a + `function_call` occurs in turn 1, exactly one `function_call_output` with the + same `call_id` occurs in turn 2, and no result precedes its call; otherwise + `fail`. +- `compaction_replayed`: return `true` iff the one + `context_compaction.encrypted_content` value is accepted into the parser's + normalized compaction slot and is absent from user-visible output; otherwise + `false`. The synthetic value is never decrypted or executed. +- `local_shell_correlated`: return `true` iff the `local_shell_call.call_id` + equals the following `function_call_output.call_id` and neither item invokes + a process; otherwise `false`. +- `tool_search_error`: for the one failed `tool_search_output`, return its exact + `error` string; missing, duplicate, or non-failed items return `null`. +- `modality_path`: return `native` when `requestHasImage` is true and + `modelInputModalities` contains `image`; otherwise return `sidecar` when an + enabled authorized vision sidecar exists; otherwise return `unsupported`. +- `silent_image_drop`: return `true` only when an image-bearing input is + omitted from adapter output without either a `native`/`sidecar` path or the + typed unsupported rejection; otherwise `false`. +- `exact_bound`: UTF-8 encode `exactSchema`; return `pass` iff its byte length + equals `limitBytes`, JSON parsing succeeds, and the complete staged catalogue + commits; otherwise `fail`. +- `one_over_rejected`: UTF-8 encode `overSchema`; return `pass` iff its byte + length equals `limitBytes + 1` and admission rejects it before commit; + otherwise `fail`. +- `partial_commit`: return `true` iff any tool from the rejected one-byte-over + staging transaction is visible in the committed catalogue; otherwise + `false`. +- `stub_received`: the in-memory MCP stub records exactly + `{namespace,name,arguments}` from the one decoded invocation. Duplicate or + missing invocations produce `null`. + +Verifier outputs use only the literal types above. A missing or type-invalid +input returns `fail`, `false`, `[]`, or `null` as specified and therefore fails +the corresponding required assertion; it is not silently repaired. + +## 8. Failure rules and freshness + +The exact ordered `protocol-v1-default` records are in the case authority. +Fixture/manifest integrity and harness failures do not affect compatibility. +Time/resource limits are environmental blockers. Exact unsupported controls +produce `UNSUPPORTED`; the exact expected rejection of a `negative_control` +satisfies that control without producing `UNSUPPORTED`; other required +deterministic mismatches are `protocol_failure`/`DEGRADED`. + +Protocol V1 scenario and suite freshness are both unbounded (`null`) because +their exact scenario, suite, fixture, compatibility-version, adapter, and +behavior digests invalidate behavior changes. A future profile may still set a +stricter maximum age. + +## 9. CL-01 boundary + +CL-01 may materialize and execute only the protocol manifests in the case +authority. It must: + +1. parse the authority as JSON and reject unknown fields; +2. recompute all fixture, scenario, and suite digests; +3. retain the exact case fixture bytes and expanded manifests + content-addressably; +4. execute only loopback mocks, closed adapter vectors, and in-memory inert MCP + stubs; +5. fail registration rather than invent semantics. + +This document and the JSON authority authorize no runner, mock server, ledger, +SQLite projection, fixture extraction from tests, live probe, or Fabric +ingestion implementation in CL-00. diff --git a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json new file mode 100644 index 000000000..087f25634 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json @@ -0,0 +1,456 @@ +{ + "schemaVersion": 1, + "authority": "CL-00 design contract; not a runtime registry", + "sourceCommit": "3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "protocol-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "negative-control-exact-rejection", "match": ["negative_control_exact_rejection"], "classification": "capability_failure", "secondaryCode": "expected_negative_control", "verdictEffect": "none", "retry": "never", "expected": true }, + { "id": "exact-unsupported", "match": ["unsupported_control_exact_rejection"], "classification": "capability_failure", "secondaryCode": "deterministic_unsupported", "verdictEffect": "unsupported", "retry": "never", "expected": true }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "protocol_conformance", + "verificationRole": "required", + "executionMode": "fixture", + "freshness": { "maxAgeMs": null }, + "executionLimits": { + "totalTimeoutMs": 10000, + "connectTimeoutMs": 1000, + "firstByteTimeoutMs": 2000, + "inactivityTimeoutMs": 2000, + "maxRequests": 4, + "maxInputBytes": 1048576, + "maxOutputBytes": 4194304, + "maxOutputTokens": 4096, + "maxToolCalls": 8, + "maxArtifactBytes": 262144 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_protocol_v1" + }, + "failureRuleSet": "protocol-v1-default" + }, + "cases": [ + { + "id": "responses-core.protocol.request-shape", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-request-shape", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"modelId\":\"fixture-model\",\"context\":{\"messages\":[{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":0}]},\"stream\":false,\"options\":{\"temperature\":0}}", "digest": "ccc7549e8bcfe4e28d0d4a87c14e622ecfb75973600b5eef830d83620c5bd0f8" }, + "assertions": [ + { "id": "method", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "message", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "PING", "required": true }, + { "id": "temperature", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/temperature", "expected": 0, "required": true } + ] + }, + { + "id": "responses-core.protocol.sse-framing", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-sse-framing-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-sse-framing", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"A\"}\n\ndata: null\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"B\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\"}}\n\n", "digest": "1c384ef32886054d8f15c14cbcbcc9af4a3bed845d6f820691368614d61515e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_text.delta", "response.output_text.delta", "response.completed"], "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "AB", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "responses-core.protocol.item-lifecycle", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-item-lifecycle-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-item-lifecycle", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[]}}\n\n", "digest": "ef271e8aaa1d63d51d4e7e0d47facadf39603c1ffa2e871865ace3684feead08" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_item.added", "response.output_item.done", "response.completed"], "required": true }, + { "id": "stable-id", "operator": "id_stable_across_events", "selector": "/client/response/events", "expected": ["/client/response/events/0/data/item/id", "/client/response/events/1/data/item/id"], "required": true }, + { "id": "id-shape", "operator": "id_matches", "selector": "/client/response/events/0/data/item/id", "expected": "responses_message", "required": true } + ] + }, + { + "id": "responses-core.protocol.terminal-state", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-terminal-state-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-terminal-state", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"fixture_failure\"}}}\n\n", "digest": "472735364ce0ee28e68192d478ccb658ec8d6a149dba6fe914e5ab35cc1a41d7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.failed"], "required": true }, + { "id": "count", "operator": "sse_event_count", "selector": "/client/response/events", "expected": { "event": "response.failed", "count": 1 }, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "responses-core.protocol.json-sse-equivalence", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http", "responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector", "raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-json-sse-equivalence", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"json\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]},\"sse\":\"event: response.output_text.delta\\ndata: {\\\"type\\\":\\\"response.output_text.delta\\\",\\\"delta\\\":\\\"OK\\\"}\\n\\nevent: response.completed\\ndata: {\\\"type\\\":\\\"response.completed\\\",\\\"response\\\":{\\\"id\\\":\\\"resp_fixture\\\",\\\"status\\\":\\\"completed\\\"}}\\n\\n\"}", "digest": "b7288170258b91361530d1dd5a0a818859ff9b6176793554ec8b0ae1177d87cf" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "equivalent", "operator": "verifier_result_equals", "selector": "/verifiers/json_sse_equivalence", "expected": "pass", "required": true } + ] + }, + { + "id": "chat-core.protocol.request-mapping", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "chat-request-mapping", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"context\":{\"systemPrompt\":[\"SYS\"],\"messages\":[{\"role\":\"developer\",\"content\":\"DEV\",\"timestamp\":0},{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":1}]},\"options\":{\"textFormat\":{\"type\":\"json_object\"}}}", "digest": "0a9c319b3a6dadbf581d0d2185f57527cd28aa57127e0eac91a421735b4c2ad9" }, + "assertions": [ + { "id": "roles", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages", "expected": [{"role":"system","content":"SYS"},{"role":"developer","content":"DEV"},{"role":"user","content":"PING"}], "required": true }, + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_object"}, "required": true } + ] + }, + { + "id": "chat-core.protocol.nonstream-envelope", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-nonstream-envelope-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "chat-nonstream-envelope", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"chatcmpl_fixture\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}", "digest": "4a9a0352daa284e73850ce613b1cc939a534d6a930c7b68c8eb28e3fcca5b248" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "done", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-assembly", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-assembly-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-assembly", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"alpha\",\"arguments\":\"{\\\"x\\\":\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"beta\",\"arguments\":\"{\\\"y\\\":\"}}]}}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"2}\"}},{\"index\":0,\"function\":{\"arguments\":\"1}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "0085f298a8690aefb74bb09ea2e0cb77703aaf6cfd822c6ce0d4d334ad4b9b3f" }, + "assertions": [ + { "id": "alpha", "operator": "tool_call_equals", "selector": "/client/response/events/0", "expected": {"id":"call_a","name":"alpha","arguments":{"x":1},"kind":"function","ordinal":0}, "required": true }, + { "id": "beta", "operator": "tool_call_equals", "selector": "/client/response/events/1", "expected": {"id":"call_b","name":"beta","arguments":{"y":2},"kind":"function","ordinal":1}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "done", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-terminal", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-terminal-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-terminal", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "done", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.request-mapping", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-request-mapping", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"system\":\"SYS\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":false}", "digest": "deeca799f660f413d0cb85263aa332bdc05aa995ccf8c7322f6af43e9bf6a627" }, + "assertions": [ + { "id": "model", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "system", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/instructions", "expected": "SYS", "required": true }, + { "id": "input", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/0/content/0/text", "expected": "PING", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.content-sequence", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-content-sequence-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-content-sequence", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["message_start","content_block_start","content_block_delta","content_block_stop","message_delta","message_stop"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.tool-round-trip", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-tool-roundtrip", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_fixture\",\"name\":\"lookup\",\"input\":{\"q\":\"x\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_fixture\",\"content\":\"RESULT\"}]}],\"tools\":[{\"name\":\"lookup\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"max_tokens\":32}", "digest": "f8dfefb427ce81fb4570f83d8d24c91e79350a7a256ecde7e636e0d70fbdff64" }, + "assertions": [ + { "id": "call-id", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/1/output", "expected": "RESULT", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.terminal-errors", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-terminal-error-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-terminal-error", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"overloaded\",\"message\":\"fixture\"}}}\n\n", "digest": "fad0d0edca35d066e89de5488635a2912930d5dedc79d047759fb6ecc6567718" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["error"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "tools-core.protocol.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-function", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tools\":[{\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"upstreamToolCall\":{\"id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"},\"toolResult\":{\"toolCallId\":\"call_fixture\",\"content\":\"RESULT\"}}", "digest": "9107f4dfdd7da8340c866c9fb6f42854437cebb98592d0510969c810c1eeb0ad" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/events/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/events/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-custom", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tool\":{\"type\":\"custom\",\"name\":\"apply_patch\",\"format\":{\"type\":\"grammar\",\"syntax\":\"lark\",\"definition\":\"start: /[\\\\s\\\\S]+/\"}},\"call\":{\"id\":\"call_patch\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\\n\"},\"output\":{\"call_id\":\"call_patch\",\"output\":\"Done\"}}", "digest": "752750104e99602d9160feaa591bcbfcfd0c8c53fc9feda4a48c3b6813b74d44" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/events/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/events/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.parallel-correlation", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["parallel_tools"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "tools-parallel-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "tools-parallel", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"b\",\"arguments\":\"{\"}}]}}]}\n\ndata:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"}\"}},{\"index\":0,\"function\":{\"arguments\":\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", "digest": "7a954d390bdf48d0dec3ed2515a5bbbedd4165656fcb7f0643ce743d17bb39f0" }, + "assertions": [ + { "id": "count", "operator": "sse_event_count", "selector": "/client/response/events", "expected": {"event":"tool_call","count":2}, "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/nonoverlap_order", "expected": ["call_a","call_b"], "required": true } + ] + }, + { + "id": "tools-core.protocol.result-content", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-result-content", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"content\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}],\"isError\":false}", "digest": "ec81d47d3d6a67254afcc21b55f458269d8dd34ab3b3d52a6c12fec9bec814ab" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "RESULT", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "tools-core.protocol.choice-and-allowed-set", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-choice", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"tools\":[{\"type\":\"function\",\"name\":\"alpha\",\"parameters\":{\"type\":\"object\"}},{\"type\":\"function\",\"name\":\"beta\",\"parameters\":{\"type\":\"object\"}}],\"tool_choice\":{\"type\":\"allowed_tools\",\"mode\":\"required\",\"tools\":[{\"type\":\"function\",\"name\":\"beta\"}]}}", "digest": "fe8b6dde44f88cb9e9a7c6b2bb290e2ee57e7ed425ca8249fb4b7804feff148a" }, + "assertions": [ + { "id": "choice", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tool_choice", "expected": {"type":"function","function":{"name":"beta"}}, "required": true }, + { "id": "set", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools", "expected": [{"type":"function","function":{"name":"beta","parameters":{"type":"object"}}}], "required": true } + ] + }, + { + "id": "codex-core.protocol.streaming-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "codex-streaming-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-streaming", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1}}\n\ndata:[DONE]\n\n", "digest": "f109d35734ecca8e71226ff739b6a0783aca283a9b0d0238a7974b2d7fd9af53" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + ] + }, + { + "id": "codex-core.protocol.apply-patch-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-patch", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** Add File: x\\n+x\\n*** End Patch\\n\",\"callId\":\"call_patch\",\"result\":\"Done\"}", "digest": "668baa1fbea1d7a6556f717467fc3b90a47b2edfaa2ccf0c7950fd30dfe27a81" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/events/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: x\n+x\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/events/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "codex-core.protocol.tool-continuation", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-tool-continuation", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"output\":[{\"type\":\"function_call\",\"id\":\"fc_fixture\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{}\"}]},\"turn2\":{\"input\":[{\"type\":\"function_call_output\",\"call_id\":\"call_fixture\",\"output\":\"RESULT\"}]}}", "digest": "0b1e955829282c51e056e0bd1d6eb88d62fbae1accd52bdda579c3fce9eac205" }, + "assertions": [ + { "id": "correlation", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/call_result_order", "expected": "pass", "required": true } + ] + }, + { + "id": "codex-core.protocol.previous-response-replay", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"stored\":{\"id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"ONE\"}],\"output\":[{\"role\":\"assistant\",\"content\":\"TWO\"}]},\"next\":{\"previous_response_id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"THREE\"}]}}", "digest": "e849a72d9772616a5ca8853bef48fd2f0884fd006b9ac747bd442513ad05e0f4" }, + "assertions": [ + { "id": "expanded", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input", "expected": [{"role":"user","content":"ONE"},{"role":"assistant","content":"TWO"},{"role":"user","content":"THREE"}], "required": true }, + { "id": "private-id", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/previous_response_id", "expected": true, "required": true } + ] + }, + { + "id": "codex-core.protocol.structured-output", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["structured_output"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-structured", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"answer\",\"schema\":{\"type\":\"object\",\"properties\":{\"ok\":{\"type\":\"boolean\"}},\"required\":[\"ok\"],\"additionalProperties\":false},\"strict\":true}}}", "digest": "e6278954535f4d482a9bb1f6c0189ef7aed00294a7bde695747b7898886cf937" }, + "assertions": [ + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_schema","json_schema":{"name":"answer","schema":{"type":"object","properties":{"ok":{"type":"boolean"}},"required":["ok"],"additionalProperties":false},"strict":true}}, "required": true } + ] + }, + { + "id": "codex-core.protocol.compaction-and-special-items", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-special-items", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"type\":\"context_compaction\",\"encrypted_content\":\"ocx1:fixture\"},{\"type\":\"local_shell_call\",\"id\":\"shell_fixture\",\"call_id\":\"call_shell\",\"status\":\"completed\",\"action\":{\"type\":\"exec\",\"command\":[\"echo\",\"ok\"]}},{\"type\":\"function_call_output\",\"call_id\":\"call_shell\",\"output\":\"ok\"},{\"type\":\"tool_search_output\",\"status\":\"failed\",\"error\":\"fixture\"}]}", "digest": "bf61cb0783f288a4dd6b0b8c0f9a2ddb60f02d0f8ea1d2875ea7e3fe1740b043" }, + "assertions": [ + { "id": "compaction", "operator": "json_path_equals", "selector": "/verifiers/compaction_replayed", "expected": true, "required": true }, + { "id": "shell", "operator": "json_path_equals", "selector": "/verifiers/local_shell_correlated", "expected": true, "required": true }, + { "id": "search", "operator": "json_path_equals", "selector": "/verifiers/tool_search_error", "expected": "fixture", "required": true } + ] + }, + { + "id": "vision-core.protocol.input-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-input", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"READ\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}]}", "digest": "a26ba5209858c3658d698c1dcb6c92845b2e6aae6bab70a7b9ad1cba1d8aa6a5" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/0/text", "expected": "READ", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/1/image_url", "expected": {"url":"data:image/png;base64,iVBORw0KGgo=","detail":"high"}, "required": true } + ] + }, + { + "id": "vision-core.protocol.tool-result-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-tool-result", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"result\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\"}]}", "digest": "02c724259bb3c98002842cafad6d890d3dab7db287f1803fd9ce97ec79630a6d" }, + "assertions": [ + { "id": "tool-text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0", "expected": {"role":"tool","tool_call_id":"call_fixture","content":"RESULT"}, "required": true }, + { "id": "image-carrier", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "vision-core.protocol.modality-gate", + "suite": "vision-core", + "capability": "modalities.image.input", + "verificationRole": "negative_control", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-gate", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"model\":\"text-only\",\"modelInputModalities\":[\"text\"],\"visionSidecar\":{\"enabled\":false},\"requestHasImage\":true}", "digest": "d5b438fb3fad873b0a1bb1b6c91539862e4f3aa8690a963eb6121c5a3229818a" }, + "assertions": [ + { "id": "path", "operator": "json_path_equals", "selector": "/verifiers/modality_path", "expected": "unsupported", "required": true }, + { "id": "no-drop", "operator": "json_path_equals", "selector": "/verifiers/silent_image_drop", "expected": false, "required": true } + ] + }, + { + "id": "reasoning-core.protocol.effort-mapping", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-effort", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"requested\":\"high\",\"reasoningEffortMap\":{\"high\":\"adaptive\"},\"reasoningWireFormat\":\"gateway-object\"}", "digest": "d9d5cce104809764d5edbc833088a0a9bb3b4d678a4f135353cc5fecf62e8b57" }, + "assertions": [ + { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"effort":"adaptive"}, "required": true }, + { "id": "legacy-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/reasoning_effort", "expected": true, "required": true } + ] + }, + { + "id": "reasoning-core.protocol.summary-stream", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "reasoning-summary-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "reasoning-summary", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.reasoning_summary_part.added\ndata: {\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"part\":{\"type\":\"summary_text\",\"text\":\"\"}}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"type\":\"response.reasoning_summary_text.delta\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"delta\":\"WHY\"}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\"}}\n\n", "digest": "d9c4fa73b67f7a92d9ec55af7ff12b16ddc9870059e5e530ee04006b171367e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.reasoning_summary_part.added","response.reasoning_summary_text.delta","response.completed"], "required": true }, + { "id": "id", "operator": "id_matches", "selector": "/client/response/events/0/data/item_id", "expected": "responses_reasoning", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/content/0/text", "expected": "PLAN", "required": true }, + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.private-content-isolation", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-private", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"origin\":{\"provider\":\"alpha\",\"encrypted\":\"opaque_fixture\"},\"destination\":{\"provider\":\"beta\",\"adapter\":\"openai-chat\"}}", "digest": "3519bd299fe2cd5b8069e0cd1c3b65b61d5ce9588c66e54b49d42af5ccf0c81e" }, + "assertions": [ + { "id": "upstream-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/encrypted_content", "expected": true, "required": true }, + { "id": "client-absent", "operator": "json_path_absent", "selector": "/client/response/json/hidden_reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.protocol.namespace-mapping", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-namespace", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"description\":\"fixture\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}}}}", "digest": "91a53f8c580d461d0f5e0d7209e5d4b95249bdfa8bd3fd4f298e18bdeadb0693" }, + "assertions": [ + { "id": "wire-name", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools/0/name", "expected": "mcp__fixture__lookup", "required": true }, + { "id": "reverse", "operator": "json_path_equals", "selector": "/client/response/events/0", "expected": {"namespace":"mcp__fixture","name":"lookup"}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.schema-and-bounds", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-bounds", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"limitBytes\":64,\"exactSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxx\\\"}\",\"overSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxxx\\\"}\"}", "digest": "34ff4414dc8e196d460390557f4fd74c32418ea00710167baff2a0dc1f3b643c" }, + "assertions": [ + { "id": "exact", "operator": "verifier_result_equals", "selector": "/verifiers/exact_bound", "expected": "pass", "required": true }, + { "id": "over", "operator": "verifier_result_equals", "selector": "/verifiers/one_over_rejected", "expected": "pass", "required": true }, + { "id": "atomic", "operator": "json_path_equals", "selector": "/verifiers/partial_commit", "expected": false, "required": true } + ] + }, + { + "id": "mcp-core.protocol.call-result", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-call", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"RESULT\"}],\"isError\":false}}", "digest": "986ef5017fbdb46eb18b30daaffe72aecc93868b7d89b11c3e244dc084f46496" }, + "assertions": [ + { "id": "call", "operator": "json_path_equals", "selector": "/verifiers/stub_received", "expected": {"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"}}, "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"RESULT"}],"isError":false}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.resource-round-trip", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-resource", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"resources\":[{\"uri\":\"fixture://one\",\"name\":\"one\"}],\"read\":{\"uri\":\"fixture://one\",\"contents\":[{\"uri\":\"fixture://one\",\"text\":\"RESOURCE\"}]}}", "digest": "a3f6317374ce92da0155dd14bbf0d5822e8687cbe8ef7968221f23acf8b16aa5" }, + "assertions": [ + { "id": "list", "operator": "json_path_equals", "selector": "/client/response/json/resources", "expected": [{"uri":"fixture://one","name":"one"}], "required": true }, + { "id": "read", "operator": "json_path_equals", "selector": "/client/response/json/contents", "expected": [{"uri":"fixture://one","text":"RESOURCE"}], "required": true } + ] + } + ] +} diff --git a/devlog/_plan/260807_compatibility_lab/030_incident_corpus.md b/devlog/_plan/260807_compatibility_lab/030_incident_corpus.md new file mode 100644 index 000000000..2e17f89e3 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/030_incident_corpus.md @@ -0,0 +1,457 @@ +# CL-00 compatibility incident corpus + +These are abstract regression specifications distilled from shipped tests, +public issues, and devlog records. Provider names identify historical evidence, +not special cases to encode in the scenario model. + +Each future fixture must reproduce the observable wire condition with a mock +upstream. CL-00 does not reopen or fix the production incidents. + +`Future mapping` identifies the suite/scenario family that should own the +regression. It does not claim protocol V1 already covers the incident. Only a +literal vector in `022_protocol_v1_cases.json` gates a V1 verdict. +Unrepresented incidents below are reviewed inputs to a later scenario/suite +version amendment; prose and source-test references cannot be inferred into +V1. + +## IC-001 - legal SSE field spacing + +- Incident class: valid SSE framing rejected. +- Historical source: + [#1170](https://github.com/lidge-jun/opencodex/issues/1170), + `devlog/_plan/260807_untouched_bug_stack/010_sse_unspaced_data_fields.md`, + `tests/sse-unspaced-data-fields.test.ts`. +- Observable failure: a parser accepts `data: {...}` but rejects legal + `data:{...}`/`event:name`, trims payload whitespace, or treats a bare + `data:` as malformed. +- Expected behavior: accept both legal forms, strip at most one optional space, + preserve payload whitespace, and handle empty field values consistently on + Responses, Chat, Anthropic and sidecar paths. +- Future mapping: `responses-core.protocol.sse-framing`, + `chat-core.protocol.stream-terminal`, + `anthropic-core.protocol.content-sequence`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; paired frames differing only in legal spacing. + +## IC-002 - null and empty SSE data frames + +- Incident class: ignorable frame mishandled as payload or terminal error. +- Historical source: + `devlog/_plan/260808_bug_campaign/020_wp2_sse_frame_contract.md`, + `tests/sse-null-data-frame.test.ts`. +- Observable failure: `data: null`, a bare `data:` field, or a comment frame + crashes decoding, creates a synthetic event, or hides a later valid event. +- Expected behavior: apply each surface's explicit ignorable-frame contract; + continue parsing without fabricating output, while malformed non-null JSON + still fails closed. +- Future mapping: `responses-core.protocol.sse-framing`, + `chat-core.protocol.stream-terminal`, + `anthropic-core.protocol.content-sequence`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; null/empty/comment/malformed controls. + +## IC-003 - missing or incorrect terminal stream signal + +- Incident class: clean EOF, `[DONE]`, completion event, and terminal state + confused. +- Historical source: `tests/openai-chat-eof.test.ts`, + `tests/sse-failed-tail.test.ts`, `tests/claude-outbound.test.ts`, + `tests/responses-stream-tool-events.test.ts`, + [#658](https://github.com/lidge-jun/opencodex/issues/658), + [#735](https://github.com/lidge-jun/opencodex/issues/735). +- Observable failure: a stream ends without the protocol-required terminal, + emits more than one terminal, accepts `[DONE]` as a Responses completion + without a terminal event, or maps failed/incomplete to successful end-turn. +- Expected behavior: exactly one surface-correct terminal. Responses remains + strict. A fingerprinted Chat/Anthropic EOF-tolerance contract may complete + only after visible output or a fully assembled tool call and only when no + incomplete call remains; every other deterministic EOF fails closed and + preserves the typed failed/incomplete reason. +- Future mapping: `responses-core.protocol.terminal-state`, + `chat-core.protocol.stream-terminal`, + `anthropic-core.protocol.terminal-errors`, + `codex-core.protocol.streaming-turn`. +- Classification: deterministic close is `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; completed, failed, incomplete, duplicate and + missing-terminal tails. + +## IC-004 - body stall versus protocol truncation + +- Incident class: environmental/provider timeout misreported as incompatibility. +- Historical source: + [#875](https://github.com/lidge-jun/opencodex/issues/875), + [#1065](https://github.com/lidge-jun/opencodex/issues/1065), + `devlog/_plan/260805_bug_stack_campaign/050_issue875_deepseek_flash_stall.md`, + `devlog/_plan/260806_overnight_triage_round2/020_bounded_body_first_byte.md`. +- Observable failure: no first byte or no later body byte arrives before the + deadline; the system labels the model's protocol unsupported, or waits + without a bound. +- Expected behavior: connect, first-byte, inactivity and total deadlines remain + distinguishable. A silent live stall is a blocker, not proof of malformed + protocol. A mock that deliberately closes without terminal data remains + IC-003. +- Future mapping: supplemental timeout controls for + `responses-core.protocol.terminal-state` and every future live suite. +- Classification: `timeout` -> `BLOCKED`; `provider_transient` when an + authoritative transient response exists. +- Deterministic fixture: yes for timeout attribution; no deterministic fixture + can convert an arbitrary live stall into incompatibility evidence. + +## IC-005 - sparse lifecycle snapshots + +- Incident class: incomplete Responses lifecycle snapshots forwarded as valid. +- Historical source: + [#893](https://github.com/lidge-jun/opencodex/issues/893), + `devlog/_plan/260805_bug_stack_campaign/040_issue893_sparse_snapshot_repair.md`, + `tests/responses-snapshot-repair.test.ts`, + `tests/responses-snapshot-repair-server.test.ts`. +- Observable failure: added/done snapshots omit required ID, type, role, status, + output index or closing item, producing a client-invalid lifecycle. +- Expected behavior: preserve a complete canonical lifecycle or apply an + explicitly configured, assertion-visible repair; never claim a sparse stream + is valid without proving the repaired output. +- Future mapping: `responses-core.protocol.item-lifecycle`, + `codex-core.protocol.streaming-turn`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; sparse permutations plus no-repair control. + +## IC-006 - invalid, reused, or missing item IDs + +- Incident class: client-facing Responses item identity violates its grammar or + event correlation. +- Historical source: + [#938](https://github.com/lidge-jun/opencodex/issues/938), + `devlog/_plan/260805_bug_stack_campaign/060_issue938_uuid_item_ids.md`, + `tests/responses-item-id-repair.test.ts`, + `tests/deepseek-responses-item-id-repair.test.ts`. +- Observable failure: UUID/placeholder/missing IDs reach a client contract that + requires typed IDs, or added/done events use inconsistent IDs. +- Expected behavior: valid stable IDs on the client surface; any configured + repair is deterministic, type-scoped, and never rewrites function call IDs + or breaks `call_id` correlation. +- Future mapping: `responses-core.protocol.item-lifecycle`, + `codex-core.protocol.streaming-turn`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; valid, invalid, reused, missing-terminal and + correlation controls. + +## IC-007 - function schema root normalization + +- Incident class: valid tool rejected because its schema root is missing or + non-object. +- Historical source: + [PR #745](https://github.com/lidge-jun/opencodex/pull/745), + `tests/responses-parser.test.ts`. +- Observable failure: tool definition reaches an object-schema-only upstream + with absent/invalid root shape, or normalization corrupts an already valid + schema. +- Expected behavior: produce the required object root without changing valid + properties/required/additionalProperties semantics. +- Future mapping: `tools-core.protocol.function-round-trip`, + `mcp-core.protocol.schema-and-bounds`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; absent, malformed and valid schema controls. + +## IC-008 - custom/freeform tool envelope mismatch + +- Incident class: function-only route or token preset rejects a valid + custom/freeform tool. +- Historical source: + `devlog/_plan/260807_untouched_bug_stack/070_mimo_token_plan_preset.md`, + `tests/responses-parser.test.ts` (exact `apply_patch` envelope), + `tests/responses-tool-groups.test.ts`. +- Observable failure: a custom tool is serialized as a function, its freeform + input/output is JSON-wrapped or dropped, or the route rejects the tool without + an honest unsupported result. +- Expected behavior: preserve the exact custom tool declaration, call and + output grammar, or deterministically classify the exact route unsupported for + custom tools. +- Future mapping: `tools-core.protocol.custom-freeform-round-trip`, + `codex-core.protocol.apply-patch-turn`. +- Classification: malformed translation is `protocol_failure` -> `DEGRADED`; + a proven route contract is `capability_failure` -> `UNSUPPORTED`. +- Deterministic fixture: yes for translation; a future live negative control + proves route support. + +## IC-009 - dangling tool calls and result correlation + +- Incident class: tool call/result pair becomes orphaned or misassociated. +- Historical source: + `devlog/_fin/260718_dangling_toolcall_hardening/010_record.md`, + `tests/openai-chat-dangling-toolcalls.test.ts`, + `tests/issue-702-expired-replay-state.test.ts`, + [#334](https://github.com/lidge-jun/opencodex/issues/334), + [#620](https://github.com/lidge-jun/opencodex/issues/620). +- Observable failure: an assistant tool call is forwarded without a matching + result, a result is attached to the wrong ID, or expired continuation state + resurrects an unrelated call. +- Expected behavior: preserve exact call/result identity and order; repair only + the narrowly declared orphan case; otherwise fail closed without fabricating + a successful tool result. +- Future mapping: `tools-core.protocol.function-round-trip`, + `codex-core.protocol.tool-continuation`, + `codex-core.protocol.previous-response-replay`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; missing, duplicate, out-of-order, expired and + mismatched IDs. + +## IC-010 - parallel tool fragment assembly + +- Incident class: interleaved calls merged, lost, reordered, or correlated to + the wrong result. +- Historical source: + `devlog/_fin/260709_parallel_tool_calls/000_plan.md`, + `tests/openai-chat-parallel-stream.test.ts`, + `tests/parallel-tool-calls-optin.test.ts`, + [#361](https://github.com/lidge-jun/opencodex/issues/361). +- Observable failure: fragmented deltas from two calls produce one argument + buffer, unstable ordering, duplicate completion, or incorrect result IDs. +- Expected behavior: assemble each indexed call independently, never duplicate + argument fragments, preserve stable order/identity, and advertise parallel + capability only when the effective adapter contract supports it. Provider + interleaving does not require overlapping canonical adapter events; atomic + sequential emission is a valid compatibility-preserving bridge contract. +- Future mapping: `tools-core.protocol.parallel-correlation`. +- Classification: `protocol_failure` -> `DEGRADED`; explicit no-parallel + contract -> `UNSUPPORTED` for that capability only. +- Deterministic fixture: yes; interleaved, fragmented, out-of-order and + single-call controls. + +## IC-011 - wrong upstream wire for a model + +- Incident class: Responses-capable and Chat-only models behind one gateway use + the provider-wide wire indiscriminately. +- Historical source: `src/types.ts` and `src/providers/registry.ts` model-wire + contract for [#404](https://github.com/lidge-jun/opencodex/issues/404), + `tests/adapter-resolve.test.ts`, `tests/deepseek-inbound-wire.test.ts`, + `tests/chat-completions-endpoint.test.ts`. +- Observable failure: an exact model is sent to the wrong endpoint/request + shape, producing rejection or silent semantic loss. +- Expected behavior: resolve the effective model-specific adapter before + subject identity and send the declared wire shape. Evidence for one wire is + never reused for the other. +- Future mapping: `responses-core.protocol.request-shape`, + `chat-core.protocol.request-mapping`; future live route variants. +- Classification: deterministic resolver/translation error is + `protocol_failure` -> `DEGRADED`; a correctly selected but unsupported route + is `capability_failure` -> `UNSUPPORTED`. +- Deterministic fixture: yes; mixed gateway with endpoint-specific fixtures. + +## IC-012 - reasoning replay form mismatch + +- Incident class: plaintext reasoning, signature, redacted block, or thought + signature is dropped or replayed in the wrong form. +- Historical source: `tests/deepseek-reasoning-replay.test.ts`, + `tests/deepseek-reasoning-replay-gaps.test.ts`, + `tests/google-antigravity-replay.test.ts`, + `tests/anthropic-thinking-signature.test.ts`, + `tests/kiro-reasoning-roundtrip.test.ts`. +- Observable failure: a second turn is rejected, reasoning text leaks into + visible output, required signature data is lost, or incompatible replay data + is forwarded. +- Expected behavior: use the exact selected adapter's replay contract, preserve + opaque data only on its compatible route, and omit/normalize it safely + elsewhere. +- Future mapping: `reasoning-core.protocol.replay`, + `reasoning-core.protocol.summary-stream`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; two-turn fixtures for each abstract replay form. + +## IC-013 - provider-private content crosses a route boundary + +- Incident class: encrypted/task/reasoning content from one provider is sent to + an incompatible provider or exposed as ordinary text. +- Historical source: + [#92](https://github.com/lidge-jun/opencodex/issues/92), + `devlog/_fin/260706_previous-response-id-400/000_plan.md`, + `tests/responses-parser.test.ts` encrypted-content case, + `tests/bridge-raw-reasoning-hidden.test.ts`, + `tests/v2-agent-message-failfast.test.ts`. +- Observable failure: opaque encrypted content is forwarded where it cannot be + decrypted, causes a 400, or becomes user-visible/private evidence. +- Expected behavior: provider-private envelopes remain origin-scoped; cross + route replay fails closed or uses a bounded opaque marker expressly allowed + by the protocol, never raw private data. +- Future mapping: `reasoning-core.protocol.private-content-isolation`, + `codex-core.protocol.previous-response-replay`; a later encrypted-task + capability scenario when its upstream contract is implementable. +- Classification: unsafe translation is `protocol_failure` -> `DEGRADED`; a + route proven unable to consume the encrypted task capability is + `UNSUPPORTED`; the current explicit fail-fast is safe `UNSUPPORTED` evidence + only when the scenario's exact route and encrypted-task preconditions match. + Raw disclosure is also a security Critical independent of compatibility + verdict. +- Deterministic fixture: yes for local origin isolation and fail-fast + mitigation; partial for true cross-provider encrypted task execution. + +## IC-014 - image modality or tool-result image mismatch + +- Incident class: structured image content is dropped, stringified, sent to a + text-only route, or advertised inaccurately. +- Historical source: + [#888](https://github.com/lidge-jun/opencodex/issues/888), + `tests/openai-chat-tool-result-images.test.ts`, + `tests/responses-parser.test.ts`, `tests/vision-anthropic.test.ts`, + `tests/vision-fail-closed.test.ts`, `tests/request-evidence.test.ts`. +- Observable failure: image order/detail/MIME is lost, a tool-result image + becomes raw JSON/text, or capability gating disagrees with the effective + native/sidecar path. +- Expected behavior: preserve structured image parts and honestly choose + native, declared sidecar, or unsupported behavior without silent loss. +- Future mapping: `vision-core.protocol.input-image`, + `vision-core.protocol.tool-result-image`, + `vision-core.protocol.modality-gate`. +- Classification: `protocol_failure` -> `DEGRADED`; proven no-image route -> + `UNSUPPORTED`; sidecar/network unavailability -> `BLOCKED`. +- Deterministic fixture: yes; synthetic data image and text-only controls. + +## IC-015 - malformed continuation and previous-response state + +- Incident class: stateful continuation is forwarded to a stateless/incompatible + route or local replay is incomplete. +- Historical source: + [#702](https://github.com/lidge-jun/opencodex/issues/702), + `devlog/_fin/260706_previous-response-id-400/000_plan.md`, + `tests/responses-state.test.ts`, + `tests/issue-702-expired-replay-state.test.ts`, + `tests/grok-orphan-adoption.test.ts`. +- Observable failure: upstream 400, duplicate history, missing prior tool call, + orphaned result, or continuation state reused after expiry/route change. +- Expected behavior: use valid provider-private continuation only on its exact + compatible subject; otherwise perform bounded ordered local expansion or fail + closed. +- Future mapping: `codex-core.protocol.previous-response-replay`, + `codex-core.protocol.tool-continuation`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; stateful, stateless, expired and route-change + matrices. + +## IC-016 - Anthropic terminal/error taxonomy corruption + +- Incident class: failed/incomplete/upstream-overload response appears as + successful `end_turn` or wrong Anthropic error type. +- Historical source: `tests/claude-outbound.test.ts`, + `tests/anthropic-eof-tolerance.test.ts`, + `tests/anthropic-compatible-stream.test.ts`. +- Observable failure: missing `message_stop` is accepted outside a declared + tolerance, transient 502 becomes a normal message, or content-filter/max-token + stop reason is mapped incorrectly. +- Expected behavior: preserve exact content-block and message terminal + sequence; map failure classes and stop reasons deterministically; apply any + EOF tolerance only to its exact fingerprinted route. +- Future mapping: `anthropic-core.protocol.terminal-errors`, + `anthropic-core.protocol.content-sequence`. +- Classification: `protocol_failure` -> `DEGRADED`; live transient -> + `provider_transient` -> `BLOCKED`. +- Deterministic fixture: yes; strict/tolerant, failed, incomplete and transient + controls. + +## IC-017 - MCP namespace, bound, and result atomicity + +- Incident class: namespace collision, oversized schema/result partial commit, + or result type loss. +- Historical source: `tests/cursor-mcp-manager.test.ts`, + `tests/cursor-mcp-stdio.test.ts`. +- Observable failure: flattened names cannot map back, one-byte-over input + leaves a partial catalogue, image/error result changes type, or unknown tool + becomes an untyped exception. +- Expected behavior: collision-safe namespace mapping, exact atomic bounds, and + typed result/error/resource behavior through a Lab-owned stub. +- Future mapping: all `mcp-core.protocol.*` scenarios. +- Classification: `protocol_failure` -> `DEGRADED`; declared no-MCP route -> + `UNSUPPORTED`. +- Deterministic fixture: yes; in-memory/loopback stub only. + +## IC-018 - DNS/connect failure poisons account or capability evidence + +- Incident class: pre-connection transport failure attributed to credentials, + account, model, or capability. +- Historical source: + [#914](https://github.com/lidge-jun/opencodex/issues/914), + `devlog/_plan/260805_bug_stack_campaign/030_issue914_dns_transport_attribution.md`, + `devlog/_plan/260803_transport_attribution/000_plan.md`, + `tests/upstream-connect-error.test.ts`. +- Observable failure: DNS/connect/TLS setup rotates account state, marks a + route capability degraded, or becomes authentication evidence. +- Expected behavior: classify pre-response transport evidence as environment/ + network, leave compatibility and credential capability unchanged, and permit + retry after environment repair. +- Future mapping: blocker controls shared by every future live suite. +- Classification: `network_failure` -> `BLOCKED`. +- Deterministic fixture: yes for attribution using an injected connect failure; + it never contributes a compatibility failure. + +## IC-019 - malformed error or empty success envelope + +- Incident class: upstream error/empty payload accepted as a successful model + response. +- Historical source: `tests/openai-chat-hardening.test.ts`, + `tests/error-fidelity.test.ts`, `tests/upstream-http-error.test.ts`. +- Observable failure: falsey error payload, empty choices, null choice, missing + message, or malformed SSE data is emitted as success or hidden by a terminal. +- Expected behavior: fail closed with a typed normalized error while preserving + any safe usage/status evidence. +- Future mapping: `chat-core.protocol.nonstream-envelope`, + `chat-core.protocol.stream-terminal`. +- Classification: `protocol_failure` -> `DEGRADED` for deterministic malformed + protocol; recognized live transient remains `provider_transient`. +- Deterministic fixture: yes. + +## IC-020 - structured-output wire mismatch + +- Incident class: Responses `text.format` is lost, malformed, or sent to an + upstream in the wrong shape. +- Historical source: `tests/responses-parser.test.ts`, + `tests/openai-chat-hardening.test.ts`, + `tests/deepseek-inbound-wire.test.ts`. +- Observable failure: JSON schema/object request widens to plain text, schema + nesting changes, or a strict unsupported route receives an invalid parameter. +- Expected behavior: preserve the known equivalent wire form, or return a + deterministic unsupported result without pretending structured output was + honored. +- Future mapping: `codex-core.protocol.structured-output`, + `responses-core.protocol.request-shape`, + `chat-core.protocol.request-mapping`. +- Classification: translation error is `protocol_failure` -> `DEGRADED`; + proven route limitation is `capability_failure` -> `UNSUPPORTED`. +- Deterministic fixture: yes for wire translation; future live negative control + for route support. + +## IC-021 - data-only Responses SSE + +- Incident class: valid Responses events rejected because the producer omits + the redundant `event:` field. +- Historical source: + [#700](https://github.com/lidge-jun/opencodex/issues/700), + `tests/claude-outbound.test.ts`. +- Observable failure: a payload with a valid typed Responses JSON record in + `data:` is ignored or treated as a truncated stream when no `event:` line is + present. +- Expected behavior: infer the event name from the payload's canonical `type` + when the surface permits data-only events, permit explicit and inferred + frames to interleave, and keep untyped data-only records ignored/fail-closed + according to the scenario. +- Future mapping: `responses-core.protocol.sse-framing`, + `anthropic-core.protocol.content-sequence`. +- Classification: `protocol_failure` -> `DEGRADED`. +- Deterministic fixture: yes; explicit-only, data-only, mixed and untyped + controls. + +## Corpus maintenance rule + +New incidents enter this corpus only when they add a reusable wire condition, +assertion, or attribution boundary. A provider-specific workaround is not a +scenario. The abstraction must state: + +```text +incident class +historical source/reference +observable failure +expected correct behavior +future scenario/suite mapping +expected failure classification +deterministic fixture feasibility +``` + +When a future fix changes the expected contract, bump the mapped scenario +version and preserve this historical record. diff --git a/devlog/_plan/260807_compatibility_lab/040_security_and_privacy.md b/devlog/_plan/260807_compatibility_lab/040_security_and_privacy.md new file mode 100644 index 000000000..08f236ae1 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/040_security_and_privacy.md @@ -0,0 +1,239 @@ +# CL-00 security, privacy, and probe sandbox contract + +Compatibility evidence is useful only if collecting it does not turn the Lab +into a data-exfiltration or arbitrary-execution surface. These requirements are +release blockers for later implementation. + +## 1. Data prohibition + +The Lab must not read, accept, persist, or export: + +- user prompts or conversation history; +- real user repositories, worktrees, patches, source files, or file paths; +- user MCP server definitions, resources, results, or credentials by default; +- arbitrary shell commands or process output; +- arbitrary filesystem contents; +- arbitrary external-network tool requests or responses; +- API keys, OAuth/access/refresh tokens, cookies, authorization material, or + raw credential errors; +- account IDs, account emails, aliases, plan labels, tenant IDs, or other PII; +- raw private/custom headers; +- hidden reasoning, chain of thought, encrypted reasoning payloads, provider + thought signatures, or decrypted private task content. + +Scenarios contain Lab-authored synthetic prompts, fixtures, tool definitions +and results only. They must be recognizable as synthetic and contain no copied +customer material. + +## 2. Future live-probe sandbox + +A live probe is an explicit background/management/CLI action. It is never +started by the production request path, profile evaluator, Router Intelligence, +request-history read, dashboard render, or provider discovery. + +The future runner must enforce a capability-deny sandbox: + +### Network + +- The immutable scenario manifest may authorize only fixed dependency roles + and protocol classes, never a route-local URL. The only remote destinations + are the exact primary endpoint and flat sidecar dependency endpoints named + in the composite route subject, after existing provider destination-policy + validation. +- DNS resolution and every redirect are revalidated. Redirects cannot widen + scheme, host, port, or private-network access. +- Private/loopback endpoints require the route's existing explicit private + network opt-in and an explicit Lab-run confirmation. Metadata endpoints + remain blocked. +- No scenario-supplied URL, model output, tool argument, or redirect may add a + destination. +- A sidecar dependency is allowed only when the scenario explicitly authorizes + its role/protocol class, the composite subject names the exact dependency + fingerprint and endpoint, the operator approves the composite live probe, + and its credential is destination-bound independently. Unmanifested roles or + subject-external/dynamically widened endpoints make the run + `harness_failure`. +- Tools have no network capability. A model-requested web search, image + generation, URL fetch, computer use, or hosted external tool is disabled or + classified inapplicable unless a future separately reviewed scenario owns a + fixed synthetic sidecar. +- Deterministic protocol tests may contact only a Lab-owned loopback mock. + +### Credentials + +- A credential broker resolves the existing route credential immediately + before the request and binds it to the validated destination. +- Credentials exist in memory for the request only and are never included in a + subject, assertion, error, artifact, log, event ID input, or SQLite row. +- Probe code receives no entire auth store and no unrelated provider/account + credential. +- Credential absence/rejection produces `authentication_blocked`, never a + compatibility failure. + +### Process and system access + +- The scenario DSL cannot express a shell command, executable, arbitrary + module, callback, script, filesystem path, or dynamic import. +- The runner receives no general shell/process API and no inherited stdin. +- Filesystem access is restricted to a fresh Lab scratch directory, read-only + packaged synthetic fixtures, and the bounded artifact writer. +- Environment inheritance is an allowlist. Secrets and proxy variables are + supplied only through reviewed destination/credential plumbing, not copied + wholesale. +- The run has enforced wall-clock, inactivity, byte, request, token, tool-call, + memory, process and artifact limits. If the platform cannot enforce a + required boundary, the run fails as `harness_failure`. +- Scratch data is deleted after artifact sanitization. Cleanup failure is + visible and retried by bounded maintenance; it does not silently retain user + data because none was admitted. + +### Tools and MCP + +- Function/custom tool scenarios expose inert Lab-authored definitions. The + harness returns static or pure-function results and never executes model + arguments. +- `apply_patch`, shell, file, browser, web-search, image-generation, computer + use and similar names are protocol tokens only. They do not invoke the real + facility. +- MCP scenarios use an in-memory or Lab-owned loopback stub with fixed schemas, + resources and pure results. User MCP configuration is not loaded. +- Cursor `nativeLocalExec`, `unsafeAllowNativeLocalExec`, desktop executors and + configured `mcpServers` are forced off for the Lab subject. Their disabled + state participates in the behavior fingerprint. + +### Agent Fabric + +- Real task execution remains in Agent Fabric's separately reviewed sandbox. +- The Lab accepts a structured outcome and sanitized content-addressed + references only. +- Outcome ingestion cannot dereference an arbitrary path or URL. Artifact + transfer uses an allowlisted broker and re-runs Lab validation. +- No task repository, prompt transcript, worktree, patch body, terminal log, or + hidden reasoning is copied into `~/.opencodex/lab/`. + +## 3. Artifact contract + +Artifacts are deny-by-default, normalized, sanitized, bounded, and +content-addressed after redaction. + +Initial hard ceilings: + +```text +maximum artifacts per run 16 +maximum bytes per artifact 256 KiB +maximum aggregate artifact data 1 MiB +maximum normalized events 4,096 +maximum sanitized string field 4 KiB +``` + +Scenario limits may be lower. Raising a hard ceiling requires a reviewed +security-contract change; a scenario manifest alone cannot raise it. + +Allowed artifact classes: + +- canonical scenario manifest; +- canonical suite manifest; +- canonical synthetic fixture; +- assertion report containing normalized expected/observed summaries; +- sanitized request shape with content replaced by type/length/digest markers; +- sanitized response shape with visible synthetic fixture output only; +- normalized bounded event trace; +- sanitized error taxonomy/status; +- deterministic verifier summary. + +Artifact paths are derived from the SHA-256 digest and fixed extension under +`~/.opencodex/lab/artifacts/`. Manifests reject traversal, symlinks, +device/special files, alternate data streams, and digest/size mismatch. The +ledger stores relative content-addressed references, never arbitrary paths. + +Scenario/suite manifests and synthetic fixtures use the domain-separated +digests in the evidence contract and remain retained while referenced by any +non-invalidated observation. Their content is still subject to the same +synthetic-data and size rules. + +Redaction occurs before hashing and writing. A redaction failure discards the +artifact and marks the run `harness_failure`; "write now, redact later" is +forbidden. + +## 4. Diagnostic sanitization + +Provider diagnostics retain only: + +- normalized HTTP status; +- allowlisted non-sensitive error type/code; +- coarse phase (`dns`, `connect`, `tls`, `first_byte`, `stream`, `terminal`); +- bounded latency/duration; +- redacted, bounded message selected by an explicit provider sanitizer. + +They remove URLs, query strings, authorization values, header dumps, request/ +response bodies, account identifiers, project/tenant names, local paths, IPs +where identifying, and token-like strings. Unknown provider diagnostics are +reduced to taxonomy and phase rather than persisted verbatim. + +Sanitizers are tested with seeded canary secrets and common credential forms. +`bun run privacy:scan` remains required but is defense in depth, not the +redaction mechanism. + +## 5. Subject privacy + +The local route subject distinguishes exact behavior without raw secrets: + +- configured instance, endpoint, custom headers, project and location use a + per-installation keyed HMAC; +- credential/account identity does not participate; +- raw base URLs and private/custom headers are absent; +- model IDs are retained locally because they are required route identity, but + custom model IDs are private-by-default for export; +- rotating the local subject salt invalidates local correlation and requires + re-projection/reverification, never reverse lookup. + +The salt is stored with secret-file permissions outside the JSONL/artifact +tree. It is not exported. + +## 6. Local evidence versus public export + +Local evidence is already sanitized. Public export is stricter and uses a new, +allowlist-only schema: + +- include suite/scenario versions, evidence layer, verdict, observation time + bucket, public registry provider/model where permitted, assertion summaries, + and public incident/scenario references; +- replace local subject/event/artifact IDs with export-scoped opaque IDs; +- omit endpoint and provider-instance fingerprints, local request/decision/ + Fabric references, precise local paths, custom headers, project/location, + custom provider/model names, account context, raw latency traces, and local + errors; +- include artifact content only when its policy explicitly says + `public_export`; local visibility does not imply export permission; +- run export-specific secret/PII scanning and fail closed on an unknown field. + +Public publishing is not authorized in CL-00 and remains a later phase. + +## 7. Retention and deletion + +- JSONL is the immutable local authority, but a user can delete the entire Lab + directory. Immutability describes in-ledger correction semantics, not a + promise to resist user deletion. +- Artifact retention classes are versioned and bounded by storage policy. + Deleting an expired artifact leaves its digest/reference and a typed + unavailable marker; it does not alter the observation. +- SQLite is disposable and contains no data absent from valid ledger events and + artifact metadata. +- Invalid or sensitive evidence is neutralized by an appended invalidation and + secure artifact deletion. A security incident may require deleting the local + ledger; append-only semantics never override the duty to remove leaked + secrets. + +## 8. Security acceptance tests required later + +Before any live runner ships, tests must prove: + +1. prompt/repository/MCP/user-tool inputs are unreachable from the scenario DSL; +2. redirects and model-supplied URLs cannot widen network access; +3. credential, account, custom header and endpoint canaries never enter + evidence, errors, SQLite or artifacts; +4. tool arguments cannot execute; +5. artifact traversal/symlink/oversize/digest attacks fail closed; +6. timeout, quota, auth, DNS and harness failures remain blockers; +7. public export rejects unknown/private fields; +8. no probe runs from the production routing path. diff --git a/devlog/_plan/260807_compatibility_lab/050_cl00_acceptance_review.md b/devlog/_plan/260807_compatibility_lab/050_cl00_acceptance_review.md new file mode 100644 index 000000000..8e70b2896 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/050_cl00_acceptance_review.md @@ -0,0 +1,147 @@ +# CL-00 independent acceptance review + +Date: 2026-08-08 + +Scope: the complete CL-00 contract set on +`feat/cl-00-compatibility-contracts`, based on +`3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296`. + +The review was read-only and separate from the authoring pass. Every validated +Critical, High, and Medium finding was corrected and re-reviewed before this +record was finalized. + +## Findings and corrections + +### Critical + +None. + +### High + +1. Initial scenario prose did not define executable selector/operator semantics + or canonical per-case manifests. + - Correction: added the closed assertion/selector/SSE contract in `021` and + the 35-case machine-readable authority in `022`, with literal fixtures, + expected values, row-specific requirements, roles, media types, limits, + artifact policy, and failure rules. +2. Immutable observations lacked enough manifest/fixture provenance to + reproduce `VERIFIED`. + - Correction: observations now carry scenario, suite, and fixture digests; + domain-separated digest preimages are exact; referenced manifests and + fixtures are retained content-addressably and cannot be replaced by the + current version during replay. +3. Route identity did not close compatibility-version and sidecar-dependent + behavior. + - Correction: froze the compatibility-version manifest/preimage, included + effective runtime and sidecar settings, added flat dependency identities, + and kept route-local endpoints in the composite subject rather than the + provider-independent scenario manifest. +4. Response-only protocol vectors did not identify an initiating client + request. + - Correction: added 11 explicit initiating request fixtures. All + `upstream_response` cases now have one request that fixes model, input, + stream mode, and inbound surface. +5. Named verifier values had no deterministic derivation. + - Correction: defined every V1 verifier as a closed pure function over the + current synthetic fixture and normalized observation. +6. Catalogue prose and incident mappings initially implied coverage beyond the + literal V1 assertions. + - Correction: narrowed every protocol V1 row to its exact `022` evidence and + made incident mappings explicit future scenario/version inputs when no + literal V1 vector exists. + +### Medium + +1. `VERIFIED -> PROBED` was missing after partial invalidation. + - Correction: added the transition for remaining partial coverage. +2. Scenario and suite freshness authorities conflicted. + - Correction: effective age is the minimum finite scenario, suite, and + profile bound. +3. Compatibility-version file hashing and dirty/missing/symlink behavior were + underspecified. + - Correction: froze the canonical object, file set, raw-byte hashes, sort + order, current-working-tree behavior, and fail-closed cases. +4. Sidecar network wording incorrectly put route-local endpoints in scenario + manifests. + - Correction: manifests authorize only dependency roles/protocol classes; + the composite subject owns exact destination fingerprints. +5. The MCP exact-bound vector was not actually at its stated boundary. + - Correction: replaced it with exact 64-byte and 65-byte UTF-8 JSON schema + payloads and a recomputed fixture digest. +6. The vision modality control could have made a compatible suite + `UNSUPPORTED`. + - Correction: made it a `negative_control`; its exact rejection satisfies + the suite without projecting route-level `UNSUPPORTED`. +7. The compaction assertion tested presence rather than truth. + - Correction: changed it to exact equality with `true`. +8. One result-content description claimed call correlation absent from its + assertions. + - Correction: removed the claim. + +### Low + +- Corrected the provider-test description: forward/static providers do not + always perform a live `/models` request. +- Corrected historical reference `#745` from issue to pull request. +- Added `021`/`022` to the stack ledger and created this review record, closing + all local document links. + +## Mechanical review evidence + +- `022_protocol_v1_cases.json` parses as JSON. +- 35 unique cases cover all required members of the eight initial suites. +- 46 fixture artifacts are present: 35 primary vectors and 11 initiating + requests. +- Every fixture digest matches + `sha256("ocx-lab:fixture:v1\0" || UTF8(bytesUtf8))`. +- Every response fixture has one initiating request. +- The MCP bound vector is exactly 64/65 UTF-8 bytes. +- All named verifier selectors have one closed deterministic definition. +- `vision-core.protocol.modality-gate` is the sole V1 negative control and is + represented as such in case and suite expansion. + +## Repository verification + +- `bun run typecheck`: passed. +- `bun run privacy:scan`: passed. +- `bun test tests/repo-hygiene.test.ts`: 11 passed, 0 failed. +- Focused protocol/compatibility suite excluding Windows privileged-symlink + state cases: 395 passed, 0 failed across 24 files. +- Focused continuation-state semantics: 2 passed, 95 filtered, 0 failed. +- Serial isolation of failures observed in the full run: + - `tests/codex-models-cache-invalidate.test.ts`: 6 passed, 0 failed. + - `tests/codex-native-residue.test.ts`: 63 passed, 2 platform skips, + 0 failed. +- Local link validation, canonical case/digest validation, and + `git diff --check`: passed. + +The full `bun run test` result is **not green**. On Windows with Bun 1.3.14 it +exited 3 after a cache-invalidation failure, an empty effective-account lookup, +and a Bun `index out of bounds` panic. A broader focused run separately found +four `responses-state.test.ts` failures, all Windows `EPERM` errors creating +symlinks (488 passed, 4 failed). The isolated cache/native tests and the +non-privileged protocol suite pass, but this review does not claim the full +suite passed. + +## Required challenge results + +1. Protocol conformance, live compatibility, and task effectiveness are + separated: **PASS**. +2. Environmental failures cannot poison compatibility verdicts: **PASS**. +3. `VERIFIED` is reproducible from immutable evidence: **PASS**. +4. Exact route identity prevents false evidence reuse: **PASS**. +5. Routing Profiles remain the sole user-policy layer: **PASS**. +6. The Lab cannot become a second router: **PASS**. +7. The Lab cannot become a second provider registry: **PASS**. +8. Probes cannot access user data or arbitrary tools: **PASS**. +9. Historical incidents are representable as deterministic versioned + scenarios: **PASS**. +10. CL-01 is implementable without semantic invention: **PASS**. + +## Verdict + +No Critical, High, or Medium findings remain. + +**CL-00: ACCEPTED** + +CL-01 remains not started and is not authorized by this review. From e0dcc2e13e4f553116fa76f6f586e7f68908819a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:51:19 +0200 Subject: [PATCH 18/77] docs(lab): record CL-00 acceptance delivery Record the accepted implementation head and draft pull request so the programme ledger is complete without authorizing CL-01. --- .../_plan/260807_compatibility_lab/001_pr_stack_status.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index e084fa66a..b72481e59 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -19,7 +19,7 @@ whether the next phase is authorized. | Phase | Branch | Base SHA | Implementation head | PR | State | |---|---|---|---|---|---| -| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | pending commit | pending | ACCEPTED | +| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `e2ea3312f83e00c0ce4c2120aaa2f86d7df2b6e4` | [draft #1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED | | CL-01 | not created | CL-00 must be accepted first | not started | none | NOT AUTHORIZED | ## CL-00 acceptance log @@ -67,6 +67,7 @@ whether the next phase is authorized. challenges pass and no Critical/High/Medium findings remain. - Blockers: none for CL-00. Full-suite green remains unavailable on this host for the Windows/Bun reasons above. -- CL-00 ending implementation SHA: pending finalization. -- Draft PR: pending. +- CL-00 ending implementation SHA: + `e2ea3312f83e00c0ce4c2120aaa2f86d7df2b6e4`. +- Draft PR: [#1286](https://github.com/lidge-jun/opencodex/pull/1286). - CL-01 authorized: **NO**. Acceptance of CL-00 does not start CL-01. From f2f2837c46a3ed20ca5d787dde7342af6d7c0fc5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:52:58 +0200 Subject: [PATCH 19/77] docs(lab): fix master plan whitespace Remove Markdown hard-break spaces so the final branch diff passes repository whitespace checks. --- devlog/_plan/260807_compatibility_lab/000_master_plan.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/000_master_plan.md b/devlog/_plan/260807_compatibility_lab/000_master_plan.md index fa41e24b4..b8483839d 100644 --- a/devlog/_plan/260807_compatibility_lab/000_master_plan.md +++ b/devlog/_plan/260807_compatibility_lab/000_master_plan.md @@ -1,7 +1,7 @@ # OpenCodex Compatibility Lab / EvalGrid -Status: CL-00 architecture authority -Authority baseline: `upstream/dev` at `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` +Status: CL-00 architecture authority +Authority baseline: `upstream/dev` at `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` Package/runtime at baseline: OpenCodex `2.10.2`, Bun `1.3.14` ## Purpose From 2cb8eddd40639b824bb2af91ce2fc53d1b998c9f Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 8 Aug 2026 22:23:53 +0900 Subject: [PATCH 20/77] fix(history): stream request-history index ingestion (#1189) (#1287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the full-tail allocation in the request-history indexer with a 64 KiB streaming reader. The previous `readCompleteTail` allocated `size - indexedOffset` bytes in one shot before parsing, so a large append created a proportional transient allocation even though the SQLite index is a disposable projection. Records are now assembled across chunk boundaries, and a complete record above 1 MiB is omitted from the projection only. `usage.jsonl` stays canonical and is never truncated or rewritten; `indexedRows` still counts successfully projected records, and the indexed offset only advances past a newline so a torn final record is re-read rather than skipped. `insert.finalize()` remains unconditional in `finally` — an unterminated prepared statement keeps the DB file busy on Windows after close. Republished from #1189 by luvs01, whose branch was 300 commits behind dev. Rebased onto f5147cbc8 with no conflicts; authorship preserved below. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- src/routing/history/indexer.ts | 93 +++++++++++++++++------------ tests/request-history-index.test.ts | 52 ++++++++++++++-- 2 files changed, 102 insertions(+), 43 deletions(-) diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index cad9ea5c5..186860fa9 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -76,6 +76,11 @@ export interface RequestHistoryPage { export const REQUEST_HISTORY_MAX_PAGE_SIZE = 100; export const REQUEST_HISTORY_DEFAULT_PAGE_SIZE = 50; export const REQUEST_HISTORY_INSERT_BATCH = 500; +export const REQUEST_HISTORY_READ_CHUNK_BYTES = 64 * 1024; +// The SQLite index is a disposable projection. Complete JSONL records above +// this bound are omitted from the projection; the canonical usage.jsonl is +// never truncated or rewritten by the indexer. +export const REQUEST_HISTORY_MAX_RECORD_BYTES = 1024 * 1024; let db: Database | null = null; let dbPath = ""; @@ -190,24 +195,6 @@ INSERT OR IGNORE INTO requests ( ?, ?, ?, ?, ? )`; -function readCompleteTail(fd: number, fromOffset: number, size: number): { text: string; nextOffset: number } { - if (size <= fromOffset) return { text: "", nextOffset: fromOffset }; - const length = size - fromOffset; - const buf = Buffer.allocUnsafe(length); - let offset = 0; - while (offset < length) { - const read = readSync(fd, buf, offset, length - offset, fromOffset + offset); - if (read === 0) throw new Error("usage log changed while indexing"); - offset += read; - } - const newline = buf.lastIndexOf(0x0a); - if (newline < 0) return { text: "", nextOffset: fromOffset }; - return { - text: buf.subarray(0, newline).toString("utf-8"), - nextOffset: fromOffset + newline + 1, - }; -} - function parsedEntryFromLine(line: string): PersistedUsageEntry | null { if (!line.trim()) return null; try { @@ -227,12 +214,11 @@ function parsedEntryFromLine(line: string): PersistedUsageEntry | null { return null; } -function ingestText(dbHandle: Database, text: string): number { - if (!text) return 0; - const lines = text.split(/\r?\n/); +function ingestSourceTail(dbHandle: Database, path: string, fromOffset: number): number { let inserted = 0; let pending: Array> = []; const insert = dbHandle.prepare(ROW_INSERT); + let fd: number | undefined; try { const commitBatch = () => { dbHandle.transaction((rows: Array>) => { @@ -243,29 +229,55 @@ function ingestText(dbHandle: Database, text: string): number { })(pending); pending = []; }; - for (const line of lines) { - const entry = parsedEntryFromLine(line); - if (!entry) continue; + const ingestLine = (line: Buffer) => { + const entry = parsedEntryFromLine(line.toString("utf-8")); + if (!entry) return; pending.push(extractRow(entry)); if (pending.length >= REQUEST_HISTORY_INSERT_BATCH) commitBatch(); - } - if (pending.length > 0) commitBatch(); - } finally { - // Windows file locks: an unterminated prepared statement keeps the DB - // file busy after close (verified on Bun 1.3.14). Finalize always. - insert.finalize(); - } - return inserted; -} + }; -function ingestSourceTail(dbHandle: Database, path: string, fromOffset: number): number { - let fd: number | undefined; - try { fd = openSync(path, "r"); const stat = fstatSync(fd); - const { text, nextOffset } = readCompleteTail(fd, fromOffset, Number(stat.size)); - if (text.length === 0 && nextOffset === fromOffset) return 0; - const inserted = ingestText(dbHandle, text); + const size = Number(stat.size); + const readBuffer = Buffer.allocUnsafe(REQUEST_HISTORY_READ_CHUNK_BYTES); + let position = fromOffset; + let nextOffset = fromOffset; + let fragments: Buffer[] = []; + let fragmentBytes = 0; + let oversized = false; + while (position < size) { + const requested = Math.min(readBuffer.length, size - position); + const bytesRead = readSync(fd, readBuffer, 0, requested, position); + if (bytesRead === 0) throw new Error("usage log changed while indexing"); + let lineStart = 0; + for (let index = 0; index < bytesRead; index++) { + if (readBuffer[index] !== 0x0a) continue; + const segment = readBuffer.subarray(lineStart, index); + if (!oversized && fragmentBytes + segment.length <= REQUEST_HISTORY_MAX_RECORD_BYTES) { + const line = fragments.length === 0 + ? segment + : Buffer.concat([...fragments, segment], fragmentBytes + segment.length); + ingestLine(line); + } + fragments = []; + fragmentBytes = 0; + oversized = false; + lineStart = index + 1; + nextOffset = position + index + 1; + } + const remainder = readBuffer.subarray(lineStart, bytesRead); + if (!oversized && fragmentBytes + remainder.length <= REQUEST_HISTORY_MAX_RECORD_BYTES) { + // Copy because readBuffer is reused on the next iteration. + fragments.push(Buffer.from(remainder)); + fragmentBytes += remainder.length; + } else if (remainder.length > 0) { + fragments = []; + fragmentBytes = 0; + oversized = true; + } + position += bytesRead; + } + if (pending.length > 0) commitBatch(); const current = readIndexedMeta(dbHandle); setMeta(dbHandle, HISTORY_META_KEYS.indexedOffset, nextOffset); setMeta(dbHandle, HISTORY_META_KEYS.indexedRows, current.indexedRows + inserted); @@ -275,6 +287,9 @@ function ingestSourceTail(dbHandle: Database, path: string, fromOffset: number): return inserted; } finally { if (fd !== undefined) closeSync(fd); + // Windows file locks: an unterminated prepared statement keeps the DB + // file busy after close (verified on Bun 1.3.14). Finalize always. + insert.finalize(); } } diff --git a/tests/request-history-index.test.ts b/tests/request-history-index.test.ts index 97d6a0a30..f67d6dd9e 100644 --- a/tests/request-history-index.test.ts +++ b/tests/request-history-index.test.ts @@ -1,5 +1,14 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs"; +import { + appendFileSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + truncateSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { handleManagementAPI } from "../src/server/management-api"; @@ -7,6 +16,7 @@ import { ManagementRequest } from "./helpers/management-auth"; import { appendUsageEntry, resetUsageReadCacheForTests, + usageLogPath, type PersistedUsageEntry, } from "../src/usage/log"; import { @@ -14,7 +24,9 @@ import { queryRequestHistory, rebuildRequestHistoryIndex, requestHistoryRowById, + REQUEST_HISTORY_MAX_RECORD_BYTES, REQUEST_HISTORY_MAX_PAGE_SIZE, + REQUEST_HISTORY_READ_CHUNK_BYTES, } from "../src/routing/history/indexer"; import { InvalidCursorError } from "../src/routing/history/cursor"; import { HISTORY_DB_FILENAME } from "../src/routing/history/schema"; @@ -194,18 +206,50 @@ describe("request-history index (RI-02)", () => { test("partial final JSONL line is skipped until it completes", async () => { for (const row of seedRows(3)) appendUsageEntry(row); + const completeOffset = statSync(usageLogPath()).size; // Append a partial line without a trailing newline. - const { appendFileSync } = await import("node:fs"); - const { usageLogPath } = await import("../src/usage/log"); appendFileSync(usageLogPath(), '{"requestId":"req-partial","timestamp":', "utf-8"); const page = await queryRequestHistory({}, undefined, 10); expect(page.rows.length).toBe(3); expect(page.meta.indexedRows).toBe(3); + expect(page.meta.indexedOffset).toBe(completeOffset); // Completing the line makes it indexable on the next refresh. appendFileSync(usageLogPath(), '9999,"provider":"a","model":"m1","status":200,"durationMs":1,"usageStatus":"reported"}\n', "utf-8"); const after = await queryRequestHistory({}, undefined, 10); expect(after.rows.length).toBe(4); - expect(after.rows.some(row => row.requestId === "req-partial")).toBe(true); + expect(after.rows.filter(row => row.requestId === "req-partial")).toHaveLength(1); + expect(after.meta.indexedOffset).toBe(statSync(usageLogPath()).size); + }); + + test("streaming refresh indexes a valid record that crosses a read chunk", async () => { + const large = entry("chunk-spanning", 9998, "a", "m1", { + apiKeyId: "x".repeat(REQUEST_HISTORY_READ_CHUNK_BYTES + 1024), + }); + const line = `${JSON.stringify(large)}\n`; + expect(Buffer.byteLength(line)).toBeGreaterThan(REQUEST_HISTORY_READ_CHUNK_BYTES); + expect(Buffer.byteLength(line)).toBeLessThan(REQUEST_HISTORY_MAX_RECORD_BYTES); + appendFileSync(usageLogPath(), line, "utf-8"); + + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.map(row => row.requestId)).toEqual(["chunk-spanning"]); + expect(page.meta.indexedOffset).toBe(statSync(usageLogPath()).size); + }); + + test("streaming refresh skips an oversized record without changing the canonical log", async () => { + const oversized = entry("oversized", 9998, "a", "m1", { + apiKeyId: "x".repeat(REQUEST_HISTORY_MAX_RECORD_BYTES + 1), + }); + const oversizedLine = `${JSON.stringify(oversized)}\n`; + expect(Buffer.byteLength(oversizedLine)).toBeGreaterThan(REQUEST_HISTORY_MAX_RECORD_BYTES); + appendFileSync(usageLogPath(), oversizedLine, "utf-8"); + appendUsageEntry(entry("after-oversized", 9999)); + const canonical = readFileSync(usageLogPath()); + + const page = await queryRequestHistory({}, undefined, 10); + expect(page.rows.map(row => row.requestId)).toEqual(["after-oversized"]); + expect(page.meta.indexedRows).toBe(1); + expect(page.meta.indexedOffset).toBe(canonical.byteLength); + expect(readFileSync(usageLogPath())).toEqual(canonical); }); test("duplicate replay is ignored", async () => { From 5aa1971123df1470bb35ec532a5161a57f1a9dc2 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 8 Aug 2026 22:23:57 +0900 Subject: [PATCH 21/77] fix(codex): warn when codex-shim install cannot prove routing (#1169) (#1289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ocx codex-shim install` reported a clean green success even when the launcher was installed but Codex routing was not provably pointed at OpenCodex — an external `model_provider`, a user-owned local or remote gateway, or routing that cannot be verified. It now reports a warning for those cases. It also warns when outbound proxy variables exist only in the current process while `config.proxy` is unset or unresolved, because Codex launchers and background services such as launchd may not inherit that environment. Proxy values are never printed. The change is advisory only: install still succeeds with the same exit code, and the shim still fail-open execs the real Codex launcher. Republished from #1169 by TyroneXie, whose branch was 335 commits behind dev. Rebased onto f5147cbc8 with no conflicts; authorship preserved below. Co-authored-by: TyroneXie <328347833@qq.com> --- .../content/docs/reference/cli/lifecycle.md | 8 + .../docs/zh-cn/reference/cli/lifecycle.md | 2 + src/cli/codex-shim-readiness.ts | 69 +++++++++ src/cli/index.ts | 9 +- tests/codex-shim-readiness.test.ts | 144 ++++++++++++++++++ 5 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 src/cli/codex-shim-readiness.ts create mode 100644 tests/codex-shim-readiness.test.ts diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index 275123528..ccf2bc18e 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -287,6 +287,14 @@ dashboard UAC prompt or rerun `ocx service install` in an elevated PowerShell wi Wrap a script-based `codex` launcher on PATH with a lightweight autostart script. Real `codex.exe` targets are left untouched to avoid breaking exact executable invocations. +Launcher installation alone does not prove that Codex requests will use OpenCodex. After a healthy +install, the command checks the current Codex routing and reports a warning instead of a green result +when routing is external, user-owned, or unverifiable. It also warns when outbound proxy variables +exist only in the current process while `config.proxy` is unset or unresolved, because Codex +launchers and background services may not inherit that environment. These checks are read-only and +never print proxy values; resolve the reported handoff and run `ocx doctor` before relying on +autostart. + If a completed external Codex update overwrites an installed shim, the next ordinary `ocx` command backs up the stable new launcher and restores the shim before dispatch. A launcher that is still changing is left untouched and retried later. Repair failures warn without failing the requested diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md index a8d64d852..81bc03406 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md @@ -178,6 +178,8 @@ ocx service uninstall 在 PATH 上把基于脚本的 `codex` 启动器包装为一个轻量自启动脚本。真实的 `codex.exe` 目标会保持不变，以避免破坏精确的可执行文件调用。 +仅安装启动器并不能证明 Codex 请求会经过 OpenCodex。完成健康安装后，命令会检查当前 Codex 路由；当路由由外部配置、用户自有网关管理或无法验证时，会显示警告而不是绿色成功。若出站代理变量只存在于当前进程，而 `config.proxy` 未设置或无法解析，也会给出警告，因为 Codex 启动器和后台服务未必继承该环境。这些检查只读且绝不会打印代理值；在依赖自动启动前，请先处理提示的交接配置并运行 `ocx doctor`。 + 如果已完成的外部 Codex 更新覆盖了已安装的 shim，下一次普通的 `ocx` 命令会先备份稳定的新启动器，再在分发前恢复 shim。仍在变动中的启动器会保持不动，并在稍后重试。修复失败只会警告，不会让所请求的命令失败；手动回退：`ocx codex-shim install`。将 `codexShimAutoRestore` 设为 `false`，或设置 `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`，即可在进程级别关闭自动恢复。 | 子命令 | 操作 | diff --git a/src/cli/codex-shim-readiness.ts b/src/cli/codex-shim-readiness.ts new file mode 100644 index 000000000..e8db24c6f --- /dev/null +++ b/src/cli/codex-shim-readiness.ts @@ -0,0 +1,69 @@ +import { + currentExternalCodexModelProvider, + getCodexRoutingKind, + type CodexRoutingKind, +} from "../codex/inject"; +import { loadConfig, resolveEnvValue } from "../config"; + +const PROXY_ENV_KEYS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", +] as const; + +export interface CodexShimReadinessInputs { + routingKind: CodexRoutingKind; + externalProvider: string | null; + processProxyEnvPresent: boolean; + configuredProxyResolved: boolean; +} + +function externalProviderLabel(provider: string | null): string { + return provider + ? `external model_provider ${JSON.stringify(provider)}` + : "the active Codex route"; +} + +export function codexShimReadinessWarnings( + inputs: CodexShimReadinessInputs, +): string[] { + const warnings: string[] = []; + const provider = externalProviderLabel(inputs.externalProvider); + + if (inputs.routingKind === "unknown") { + warnings.push( + inputs.externalProvider + ? `Codex still selects ${provider}. The shim can start OpenCodex, but it does not redirect that provider; point it at the live OpenCodex /v1 endpoint with wire_api = "responses", or switch to the built-in openai provider and run 'ocx sync'.` + : "Codex routing could not be verified. The shim can start OpenCodex, but it may not redirect Codex; run 'ocx doctor' before relying on autostart.", + ); + } else if (inputs.routingKind === "custom-local") { + warnings.push( + `Codex uses ${provider} through a user-owned local gateway. The shim can start OpenCodex, but OpenCodex does not own that route; run 'ocx doctor' to verify its lifecycle.`, + ); + } else if (inputs.routingKind === "custom-remote") { + warnings.push( + `Codex uses ${provider} through a remote gateway. The shim only starts a local OpenCodex proxy and will not affect those requests.`, + ); + } + + if (inputs.processProxyEnvPresent && !inputs.configuredProxyResolved) { + warnings.push( + "Proxy environment variables are present only in this process while config.proxy is unset or unresolved. Codex launchers and background services may not inherit them; persist config.proxy before relying on autostart.", + ); + } + + return warnings; +} + +export function collectCodexShimReadinessWarnings(): string[] { + const config = loadConfig(); + return codexShimReadinessWarnings({ + routingKind: getCodexRoutingKind(), + externalProvider: currentExternalCodexModelProvider(), + processProxyEnvPresent: PROXY_ENV_KEYS.some(key => Boolean(process.env[key]?.trim())), + configuredProxyResolved: Boolean(resolveEnvValue(config.proxy)?.trim()), + }); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index a66f0a4e3..d96b128f9 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1160,11 +1160,16 @@ switch (command) { break; } case "codex-shim": { - const { codexShimStatus, installCodexShim, uninstallCodexShim } = await import("../codex/shim"); + const { codexShimStatus, diagnoseCodexShim, installCodexShim, uninstallCodexShim } = await import("../codex/shim"); switch (args[1]) { case "install": { const r = installCodexShim(); - console.log(r.installed ? `✅ ${r.message}` : `⚠️ ${r.message}`); + const { collectCodexShimReadinessWarnings } = await import("./codex-shim-readiness"); + const warnings = diagnoseCodexShim().healthy + ? collectCodexShimReadinessWarnings() + : []; + console.log(`${r.installed && warnings.length === 0 ? "✅ " : "⚠️ "}${r.message}`); + for (const warning of warnings) console.warn(` ${warning}`); break; } case "status": diff --git a/tests/codex-shim-readiness.test.ts b/tests/codex-shim-readiness.test.ts new file mode 100644 index 000000000..7c004aef9 --- /dev/null +++ b/tests/codex-shim-readiness.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { codexShimReadinessWarnings } from "../src/cli/codex-shim-readiness"; + +const repoRoot = dirname(fileURLToPath(new URL("../package.json", import.meta.url))); +const cliPath = join(repoRoot, "src", "cli", "index.ts"); + +const ready = { + routingKind: "native" as const, + externalProvider: null, + processProxyEnvPresent: false, + configuredProxyResolved: false, +}; + +describe("Codex shim install readiness", () => { + test("keeps a clean install green for native and managed routing", () => { + expect(codexShimReadinessWarnings(ready)).toEqual([]); + expect(codexShimReadinessWarnings({ + ...ready, + routingKind: "opencodex-local", + })).toEqual([]); + }); + + test("warns when an external provider is not routed through OpenCodex", () => { + const warnings = codexShimReadinessWarnings({ + ...ready, + routingKind: "unknown", + externalProvider: "custom", + }); + + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain('external model_provider "custom"'); + expect(warnings[0]).toContain("live OpenCodex /v1 endpoint"); + expect(warnings[0]).toContain('wire_api = "responses"'); + + }); + + test("distinguishes user-owned local and remote routes", () => { + const local = codexShimReadinessWarnings({ + ...ready, + routingKind: "custom-local", + externalProvider: "gateway", + }); + expect(local).toHaveLength(1); + expect(local[0]).toContain("user-owned local gateway"); + expect(local[0]).toContain("ocx doctor"); + + const remote = codexShimReadinessWarnings({ + ...ready, + routingKind: "custom-remote", + externalProvider: "gateway", + }); + expect(remote).toHaveLength(1); + expect(remote[0]).toContain("remote gateway"); + expect(remote[0]).toContain("will not affect those requests"); + }); + + test("warns about process-only proxy settings without exposing a URL", () => { + const warnings = codexShimReadinessWarnings({ + ...ready, + processProxyEnvPresent: true, + configuredProxyResolved: false, + }); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("config.proxy"); + expect(warnings[0]).toContain("may not inherit"); + expect(warnings[0]).not.toContain("://"); + + expect(codexShimReadinessWarnings({ + ...ready, + processProxyEnvPresent: true, + configuredProxyResolved: true, + })).toEqual([]); + }); + + test("the install command surfaces readiness warnings without leaking the proxy URL", () => { + if (process.platform === "win32") return; + + const root = mkdtempSync(join(tmpdir(), "ocx-shim-readiness-")); + const codexHome = join(root, "codex-home"); + const opencodexHome = join(root, "opencodex-home"); + const binDir = join(root, "bin"); + mkdirSync(codexHome); + mkdirSync(opencodexHome); + mkdirSync(binDir); + try { + writeFileSync(join(codexHome, "config.toml"), [ + 'model_provider = "custom"', + "", + "[model_providers.custom]", + 'name = "OpenAI"', + 'wire_api = "responses"', + "", + ].join("\n"), "utf8"); + writeFileSync(join(opencodexHome, "config.json"), `${JSON.stringify({ + port: 10100, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + }, + }, + defaultProvider: "openai", + }, null, 2)}\n`, "utf8"); + const codex = join(binDir, "codex"); + writeFileSync(codex, "#!/bin/sh\nexit 0\n", "utf8"); + chmodSync(codex, 0o755); + + const proxyUrl = "http://user:secret@127.0.0.1:7890"; + const result = spawnSync(process.execPath, [cliPath, "codex-shim", "install"], { + cwd: repoRoot, + env: { + ...process.env, + CODEX_HOME: codexHome, + OPENCODEX_HOME: opencodexHome, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + HTTP_PROXY: proxyUrl, + HTTPS_PROXY: proxyUrl, + }, + encoding: "utf8", + }); + + expect(result.status).toBe(0); + expect(result.stdout).toStartWith("⚠️ Codex autostart shim installed"); + expect(result.stderr).toContain('external model_provider "custom"'); + expect(result.stderr).toContain("config.proxy"); + expect(`${result.stdout}\n${result.stderr}`).not.toContain(proxyUrl); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("user:secret"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 5222f354aa5d848cdc40735cde0a83bd3db0f791 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 8 Aug 2026 22:35:01 +0900 Subject: [PATCH 22/77] fix(quota): prefer Grok weekly credits for xAI dashboard (#1290) OpenCodex still showed the legacy 30-day Grok billing window while the real SuperGrok gate is weekly. Prefer GET /v1/billing?format=credits and keep monthly /v1/billing only as fallback when weekly data is unavailable. Closes #1283 --- .../000_plan.md | 109 +++++++++++++ src/providers/quota.ts | 112 +++++++++++--- tests/provider-quota.test.ts | 143 +++++++++++++++++- 3 files changed, 344 insertions(+), 20 deletions(-) create mode 100644 devlog/_plan/260808_issue1283_xai_weekly_quota/000_plan.md diff --git a/devlog/_plan/260808_issue1283_xai_weekly_quota/000_plan.md b/devlog/_plan/260808_issue1283_xai_weekly_quota/000_plan.md new file mode 100644 index 000000000..ec0424fcf --- /dev/null +++ b/devlog/_plan/260808_issue1283_xai_weekly_quota/000_plan.md @@ -0,0 +1,109 @@ +--- +created: 2026-08-08 +status: active +tags: [xai, grok, quota, issue-1283, dashboard] +--- + +# Issue #1283 — Grok dashboard weekly quota (OpenCodex) + +## Loop spec + +- Archetype: spec-satisfaction repair +- Trigger: https://github.com/lidge-jun/opencodex/issues/1283 reports dashboard shows 30-day/monthly Grok usage while Codex/Grok CLI gates on weekly limit. +- Goal: OpenCodex provider quota for OAuth `xai` prefers Grok weekly credits and only falls back to legacy monthly billing when weekly data is unavailable. +- Non-goals: ima2-gen/cli-jaw changes; multi-account xAI pool aggregation redesign; docs-site locale churn; release/version bump; secret-store redesign. +- Verifier: `bun test tests/provider-quota.test.ts`; `bun run typecheck` if types change; `git diff --check`. +- Stop: weekly-first path + monthly fallback covered by focused tests; branch pushed; PR targets `dev` with `Closes #1283`. +- Terminal outcomes: DONE on green tests + PR; NOOP only if tree already weekly-first; BLOCKED only if contract cannot be determined. + +## Evidence already known + +- OpenCodex `fetchXaiQuota` (`src/providers/quota.ts`) still calls `GET https://cli-chat-proxy.grok.com/v1/billing` and maps `monthlyLimit/used` → `monthlyPercent` (introduced 2026-07-05, unchanged). +- Prior cross-repo work (2026-07-16) moved **ima2-gen** (and intended cli-jaw) to `GET /v1/billing?format=credits` with envelope `{ config: { creditUsagePercent?, currentPeriod: { type: USAGE_PERIOD_TYPE_WEEKLY, end } } }`. +- OpenCodex already has: + - `XAI_GROK_COMPATIBILITY` client headers in `src/providers/xai-transport.ts` + - `credential.accountId` from JWT `sub` / Grok CLI `user_id` (`src/oauth/xai.ts`, `src/oauth/local-token-detect.ts`) + - `getCredential("xai")` / `getValidAccessToken("xai")` for the active OAuth account + +## Diff-level plan + +### IN + +1. MODIFY `src/providers/quota.ts` + - Add constants: + - `XAI_BILLING_URL = "https://cli-chat-proxy.grok.com/v1/billing"` + - `XAI_CREDITS_URL = XAI_BILLING_URL + "?format=credits"` + - Add pure parser `parseXaiCreditsResponse(value: unknown): { percent: number; resetAt?: number } | null` + - Require `config.currentPeriod.type === "USAGE_PERIOD_TYPE_WEEKLY"` and parseable `end` + - `creditUsagePercent` optional: omit → `0`; non-finite number → reject + - Clamp via `normalizePercent` + - Rewrite `fetchXaiQuota(provider)`: + 1. Resolve access token via `getValidAccessToken("xai")`; on failure return null. + 2. Read active credential via `getCredential("xai")` for optional `accountId`. + 3. If `accountId` is non-empty, attempt weekly credits request with headers: + - `Accept: application/json` + - `Authorization: Bearer ` + - `x-xai-token-auth: xai-grok-cli` + - `x-authenticateresponse: authenticate-response` + - `x-userid: ` + - `x-grok-client-version: XAI_GROK_CLIENT_VERSION` + - Prefer importing constants from `./xai-transport` rather than duplicating version. + - Isolate throw/non-2xx/malformed/non-weekly into null so monthly can run. + - On success return `report(provider, "xai:grok-billing-credits", { weeklyPercent, weeklyResetAt?, updatedAt })`. + 4. Legacy monthly fallback: current bare `/v1/billing` parse of `monthlyLimit/used` → `monthlyPercent`/`monthlyResetAt`, source remains `xai:grok-billing`. + - Do not log tokens, user ids, or raw body fields. + +2. MODIFY `tests/provider-quota.test.ts` + - Existing multi-provider fixture currently mocks only bare `/v1/billing` and expects `xai.monthlyPercent === 25`. Keep that path green: either leave accountId absent so weekly is skipped, or answer weekly with non-weekly/null and still serve monthly. + - Add focused xAI cases: + - weekly success: credential with `accountId`, credits URL returns weekly envelope → `weeklyPercent` + `weeklyResetAt` + source `xai:grok-billing-credits`; assert request URL ends with `format=credits` and required headers present without asserting secret values beyond bearer token already used. + - omitted percent → weekly 0 + - weekly non-2xx / malformed / non-weekly period → monthly fallback still works + - missing accountId → skip weekly, monthly only + - Keep privacy assertions: report JSON must not include access secrets / raw_secret fields. + +### OUT + +- GUI component rewrite (weekly bar already renders when `weeklyPercent` is present) +- Changing Codex WHAM weekly/monthly plan logic +- Live network smoke requiring real Grok auth (optional only) + +## Activation scenarios (C) + +1. Weekly non-zero path fires when mock returns `USAGE_PERIOD_TYPE_WEEKLY` + percent. +2. Weekly zero-omission path fires when percent key omitted. +3. Fallback activation: rejected weekly response still yields monthly percent from second call. +4. Missing identity skips credits URL entirely. + +## Verification commands + +```bash +bun test tests/provider-quota.test.ts +bun run typecheck +git diff --check +``` + +## Publish + +- Branch: `codex/260808-1283-xai-weekly-quota` +- Commit message: `fix(quota): prefer Grok weekly credits for xAI dashboard` +- Push and open PR to `dev` with template + `Closes #1283` (user authorized push). + + +## Audit synthesis — round 1 (main, 2026-08-08) + +Independent reviewer dispatch timed out with empty output; main agent performed the adversarial read-only audit against local code and ima2-gen. + +Accepted amendments before B: + +1. **Header case / constants (High):** use `XAI_GROK_COMPATIBILITY.headers.tokenAuth` / `authenticateResponse` / `clientVersion` and `XAI_GROK_CLIENT_VERSION` from `src/providers/xai-transport.ts`. Do not invent mixed-case aliases. Keep `x-userid` literal as in ima2-gen weekly path (not present on chat transport). +2. **Identity resolution (High):** read `getCredential("xai")?.accountId` first; if missing, decode JWT `sub` from the active access token the same way `getTokenIdentity` does (base64url payload). Missing identity skips weekly and falls back monthly. +3. **Client version (Medium):** use OpenCodex pinned `XAI_GROK_CLIENT_VERSION` rather than reading `~/.grok/version.json`. OpenCodex chat transport already pins this; weekly quota should match product identity, not require a local Grok CLI install. +4. **Percent semantics (Medium):** use `normalizePercent` (clamp, no Math.round) for consistency with other provider quota parsers in this file; tests must not assume integer rounding of 12.3. +5. **Source labels (Low):** weekly success → `xai:grok-billing-credits`; monthly fallback → `xai:grok-billing`. +6. **Exception isolation (High):** weekly attempt must catch network/JSON/parse failures and continue to monthly; never throw out of `fetchXaiQuota`. +7. **Fixture preservation (High):** existing multi-provider fixture saves credentials without accountId and mocks only bare `/v1/billing`; keep weekly skip-on-missing-identity so `monthlyPercent: 25` stays green. Focused weekly tests use credentials with accountId. +8. **No dual-window merge in v1:** when weekly succeeds, return weekly only (the gating window). Do not also attach stale monthly from a second call in the success path. +9. **Privacy:** never put accountId/user id into report objects or logs. + +VERDICT: GO-WITH-FIXES (blockers=4 High folded into plan above) diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 01516340a..b253c27fe 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -11,6 +11,7 @@ import { getValidAccessToken, getValidAccessTokenForAccount } from "../oauth"; import { getAccountCredential, getAccountSet, getCredential } from "../oauth/store"; import { antigravityUserAgent } from "../adapters/client-fingerprint"; import { apiKeyPoolEntryId } from "./api-keys"; +import { XAI_GROK_CLIENT_VERSION, XAI_GROK_COMPATIBILITY } from "./xai-transport"; import { getProviderRegistryEntry, providerCodexAccountMode } from "./registry"; import type { OcxConfig, OcxProviderConfig } from "../types"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "./openai-tiers"; @@ -44,6 +45,8 @@ const VENICE_BASE_URL = "https://api.venice.ai/api/v1"; const SYNTHETIC_BASE_URL = "https://api.synthetic.new/v2"; const DEEPINFRA_BASE_URL = "https://api.deepinfra.com"; const NEURALWATT_BASE_URL = "https://api.neuralwatt.com/v1"; +const XAI_BILLING_URL = "https://cli-chat-proxy.grok.com/v1/billing"; +const XAI_CREDITS_URL = `${XAI_BILLING_URL}?format=credits`; /** Keep a failed probe's previous row at most this long before dropping it. */ const LAST_GOOD_MAX_AGE_MS = CODEX_CAPACITY_MAX_QUOTA_AGE_MS; const nativeMainReportGenerations = new WeakMap(); @@ -977,6 +980,65 @@ function centsValue(value: unknown): number | undefined { return rec ? toFiniteNumber(rec.val) : undefined; } +/** Decode JWT payload `sub` for xAI weekly credits when the stored credential lacks accountId. */ +function xaiUserIdFromAccessToken(accessToken: string): string | undefined { + const parts = accessToken.split("."); + if (parts.length < 2 || !parts[1]) return undefined; + try { + const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as { sub?: unknown }; + return typeof payload.sub === "string" && payload.sub.trim() ? payload.sub.trim() : undefined; + } catch { + return undefined; + } +} + +/** + * Grok Build weekly credits envelope: + * `{ config: { creditUsagePercent?, currentPeriod: { type: "USAGE_PERIOD_TYPE_WEEKLY", end } } }`. + * Omitted percent is treated as 0 (proto3 default). + */ +export function parseXaiCreditsResponse(value: unknown): { percent: number; resetAt?: number } | null { + const body = asRecord(value); + const config = asRecord(body?.config); + if (!config) return null; + const period = asRecord(config.currentPeriod); + if (!period || period.type !== "USAGE_PERIOD_TYPE_WEEKLY") return null; + const resetAt = normalizeResetAt(period.end); + if (resetAt === undefined) return null; + if (config.creditUsagePercent !== undefined) { + const percent = normalizePercent(config.creditUsagePercent); + if (percent === undefined) return null; + return { percent, resetAt }; + } + return { percent: 0, resetAt }; +} + +async function fetchXaiWeeklyCredits(accessToken: string, userId: string): Promise { + try { + const response = await fetch(XAI_CREDITS_URL, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${accessToken}`, + [XAI_GROK_COMPATIBILITY.headers.tokenAuth]: "xai-grok-cli", + [XAI_GROK_COMPATIBILITY.headers.authenticateResponse]: "authenticate-response", + "x-userid": userId, + [XAI_GROK_COMPATIBILITY.headers.clientVersion]: XAI_GROK_CLIENT_VERSION, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!response.ok) return null; + const parsed = parseXaiCreditsResponse(await response.json().catch(() => null)); + if (!parsed) return null; + return { + weeklyPercent: parsed.percent, + ...(parsed.resetAt !== undefined ? { weeklyResetAt: parsed.resetAt } : {}), + updatedAt: Date.now(), + }; + } catch { + return null; + } +} + async function fetchXaiQuota(provider: string): Promise { let accessToken: string; try { @@ -984,25 +1046,37 @@ async function fetchXaiQuota(provider: string): Promise null)); - const config = asRecord(body?.config); - if (!config) return null; - const limitCents = centsValue(config.monthlyLimit); - const usedCents = centsValue(config.used); - if (limitCents === undefined || usedCents === undefined || limitCents <= 0) return null; - const percent = normalizePercent((usedCents / limitCents) * 100); - if (percent === undefined) return null; - const quota: ProviderQuota = { - monthlyPercent: percent, - monthlyResetAt: normalizeResetAt(config.billingPeriodEnd), - updatedAt: Date.now(), - }; - return report(provider, "xai:grok-billing", quota); + + // Prefer the SuperGrok weekly credits window that actually gates prompting (#1283). + const userId = getCredential("xai")?.accountId?.trim() || xaiUserIdFromAccessToken(accessToken); + if (userId) { + const weekly = await fetchXaiWeeklyCredits(accessToken, userId); + if (weekly) return report(provider, "xai:grok-billing-credits", weekly); + } + + // Legacy monthly dollar pool — retained when weekly is unavailable. + try { + const response = await fetch(XAI_BILLING_URL, { + headers: { Accept: "application/json", Authorization: `Bearer ${accessToken}` }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!response.ok) return null; + const body = asRecord(await response.json().catch(() => null)); + const config = asRecord(body?.config); + if (!config) return null; + const limitCents = centsValue(config.monthlyLimit); + const usedCents = centsValue(config.used); + if (limitCents === undefined || usedCents === undefined || limitCents <= 0) return null; + const percent = normalizePercent((usedCents / limitCents) * 100); + if (percent === undefined) return null; + return report(provider, "xai:grok-billing", { + monthlyPercent: percent, + monthlyResetAt: normalizeResetAt(config.billingPeriodEnd), + updatedAt: Date.now(), + }); + } catch { + return null; + } } function parseClaudeBucket(value: unknown): { percent?: number; resetAt?: number } | null { diff --git a/tests/provider-quota.test.ts b/tests/provider-quota.test.ts index 4a0ed3b1c..7d58f1819 100644 --- a/tests/provider-quota.test.ts +++ b/tests/provider-quota.test.ts @@ -12,10 +12,10 @@ import { saveCredential } from "../src/oauth/store"; import { clearProviderQuotaCache, fetchProviderQuotaReports, + parseXaiCreditsResponse, setProviderQuotaBeforePublishForTests, } from "../src/providers/quota"; import type { OcxConfig } from "../src/types"; - const originalFetch = globalThis.fetch; const previousOpencodexHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; @@ -1864,6 +1864,147 @@ describe("fetchProviderQuotaReports", () => { await nonForced; }); + + test("parseXaiCreditsResponse maps weekly credits and rejects non-weekly periods", () => { + expect(parseXaiCreditsResponse({ + config: { + creditUsagePercent: 57.4, + currentPeriod: { type: "USAGE_PERIOD_TYPE_WEEKLY", end: "2026-08-15T13:05:52.277209Z" }, + }, + })).toEqual({ + percent: 57.4, + resetAt: Date.parse("2026-08-15T13:05:52.277209Z"), + }); + expect(parseXaiCreditsResponse({ + config: { + currentPeriod: { type: "USAGE_PERIOD_TYPE_WEEKLY", end: "2026-08-15T13:05:52.277209Z" }, + }, + })).toEqual({ + percent: 0, + resetAt: Date.parse("2026-08-15T13:05:52.277209Z"), + }); + expect(parseXaiCreditsResponse({ + config: { + creditUsagePercent: 10, + currentPeriod: { type: "USAGE_PERIOD_TYPE_MONTHLY", end: "2026-08-15T13:05:52.277209Z" }, + }, + })).toBeNull(); + }); + + test("xAI OAuth quota prefers weekly credits and falls back to monthly when weekly fails", async () => { + await saveCredential("xai", { + access: "xai-access-secret", + refresh: "xai-refresh-secret", + expires: Date.now() + 3600_000, + accountId: "xai-user-1", + }); + const seen: { url: string; headers: Record }[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const headers = Object.fromEntries(new Headers(init?.headers).entries()); + seen.push({ url, headers }); + if (url === "https://cli-chat-proxy.grok.com/v1/billing?format=credits") { + return new Response(JSON.stringify({ + config: { + creditUsagePercent: 31, + currentPeriod: { type: "USAGE_PERIOD_TYPE_WEEKLY", end: "2026-08-15T00:00:00Z" }, + raw_secret_should_not_escape: "xai-access-secret", + }, + }), { status: 200, headers: { "content-type": "application/json" } }); + } + if (url === "https://cli-chat-proxy.grok.com/v1/billing") { + return new Response(JSON.stringify({ + config: { + monthlyLimit: { val: 10_000 }, + used: { val: 2_500 }, + billingPeriodEnd: "2026-08-31T00:00:00Z", + }, + }), { status: 200, headers: { "content-type": "application/json" } }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + const weekly = await fetchProviderQuotaReports({ + defaultProvider: "xai", + providers: { xai: { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1" } }, + } as OcxConfig, true); + expect(weekly.reports).toHaveLength(1); + expect(weekly.reports[0]?.source).toBe("xai:grok-billing-credits"); + expect(weekly.reports[0]?.quota).toMatchObject({ + weeklyPercent: 31, + weeklyResetAt: Date.parse("2026-08-15T00:00:00Z"), + }); + expect(weekly.reports[0]?.quota.monthlyPercent).toBeUndefined(); + const creditsCall = seen.find(row => row.url.endsWith("format=credits")); + expect(creditsCall?.headers.authorization).toBe("Bearer xai-access-secret"); + expect(creditsCall?.headers["x-userid"]).toBe("xai-user-1"); + expect(creditsCall?.headers["x-xai-token-auth"]).toBe("xai-grok-cli"); + expect(creditsCall?.headers["x-authenticateresponse"]).toBe("authenticate-response"); + expect(creditsCall?.headers["x-grok-client-version"]).toBeTruthy(); + expect(JSON.stringify(weekly)).not.toContain("xai-access-secret"); + expect(JSON.stringify(weekly)).not.toContain("xai-user-1"); + + // Weekly non-2xx falls back to monthly. + seen.length = 0; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const headers = Object.fromEntries(new Headers(init?.headers).entries()); + seen.push({ url, headers }); + if (url.endsWith("format=credits")) { + return new Response("nope", { status: 503 }); + } + if (url === "https://cli-chat-proxy.grok.com/v1/billing") { + return new Response(JSON.stringify({ + config: { + monthlyLimit: { val: 10_000 }, + used: { val: 2_500 }, + billingPeriodEnd: "2026-08-31T00:00:00Z", + }, + }), { status: 200, headers: { "content-type": "application/json" } }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + const monthly = await fetchProviderQuotaReports({ + defaultProvider: "xai", + providers: { xai: { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1" } }, + } as OcxConfig, true); + expect(monthly.reports[0]?.source).toBe("xai:grok-billing"); + expect(monthly.reports[0]?.quota.monthlyPercent).toBe(25); + expect(monthly.reports[0]?.quota.weeklyPercent).toBeUndefined(); + expect(seen.some(row => row.url.endsWith("format=credits"))).toBe(true); + expect(seen.some(row => row.url === "https://cli-chat-proxy.grok.com/v1/billing")).toBe(true); + }); + + test("xAI OAuth quota skips weekly when identity is absent and keeps monthly", async () => { + await saveCredential("xai", { + access: "xai-access-secret", + refresh: "xai-refresh-secret", + expires: Date.now() + 3600_000, + }); + const seen: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + seen.push(url); + if (url === "https://cli-chat-proxy.grok.com/v1/billing") { + return new Response(JSON.stringify({ + config: { + monthlyLimit: { val: 10_000 }, + used: { val: 2_500 }, + billingPeriodEnd: "2026-08-31T00:00:00Z", + }, + }), { status: 200, headers: { "content-type": "application/json" } }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + const result = await fetchProviderQuotaReports({ + defaultProvider: "xai", + providers: { xai: { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1" } }, + } as OcxConfig, true); + expect(seen.some(url => url.includes("format=credits"))).toBe(false); + expect(result.reports[0]?.source).toBe("xai:grok-billing"); + expect(result.reports[0]?.quota.monthlyPercent).toBe(25); + }); + test("interleaved configs keep independent inflight entries (A → B → A joins the first A)", async () => { await saveCredential("cursor", { access: "cursor-access-secret", refresh: "cursor-refresh-secret", expires: Date.now() + 3600_000 }); await saveCredential("xai", { access: "xai-access-secret", refresh: "xai-refresh-secret", expires: Date.now() + 3600_000 }); From 57ea8df472c33355cb7380e92fc8e99ab139aa53 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 8 Aug 2026 22:48:10 +0900 Subject: [PATCH 23/77] fix(routing): keep unbound account quota unknown (#1195) (#1288) * fix(routing): keep unbound account quota unknown (#1195) Policy profiles choose a provider and model before the request path resolves Pool/Direct identity, thread affinity, Anthropic session affinity, or round-robin/fill-first selection. Attaching the process-global active account during policy evaluation could therefore score or exclude a candidate using account A's quota and then execute the request on account B. An unbound candidate now stays quota-unknown in both the live route trace and the management dry-run, which is more accurate than inventing an account reference and keeps account selection, cooldowns, and session affinity authoritative. Unknown quota already has an explicit profile policy. Explicit `codexAccountId` and account-ref evidence remains unchanged. Republished from #1195 by luvs01, whose branch was 300 commits behind dev. Rebased onto f5147cbc8 with no conflicts; authorship preserved below. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> * test(routing): prove the management dry-run leaves unbound candidates unknown Maintainer-added coverage for the #1195 republish. The contributor's patch deletes the same block from the live router and the management dry-run path, but only the live path had a regression. The existing dry-run test covers a candidate with an explicitly supplied codexAccountId, which stays known and is unaffected by the fix, so the dry-run half of the parity claim was unproven. These two tests exercise an unbound Codex candidate with an active pool account, and an unbound Anthropic candidate with an active account, and assert both stay quota-unknown with no accountRef. Restoring either deleted block fails them. --------- Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- .../docs/reference/configuration/routing.md | 20 ++--- src/router.ts | 20 ----- .../management/routing-profile-routes.ts | 22 ------ tests/quota-scoring.test.ts | 40 +++++++++- tests/routing-profile.test.ts | 75 +++++++++++++++++++ 5 files changed, 123 insertions(+), 54 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 862dd9867..d51e152ad 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -153,15 +153,17 @@ CLI: `ocx route policy list [--json]`, `ocx route policy show [--json]`, an Dry-run evaluates candidates without sending any upstream request. Quota evidence (`optimize.quota`, `require.minQuotaHeadroom`, `unknownEvidence.quota`) comes from -the local Codex pool and Anthropic account quota caches, which are keyed by account. In **Pool** mode -the canonical `openai` provider preserves its existing account selection, then reads quota for the -selected account; **Direct** mode reads quota only from the current (caller/main) account. For other -providers (e.g. Anthropic), runtime candidates use the provider's active account. Quota evidence -never changes account selection, session affinity, cooldowns, or switching behavior — it only feeds -policy scoring. To see quota-aware behavior in a dry-run, supply account refs through the dry-run/API -candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai`) or -`candidates[].accountRef` (Anthropic) derives the matching cached account quota; an explicit -`candidates[].quota` object is echoed as given. +account-keyed Codex and Anthropic quota caches. A runtime candidate receives cached quota only when +the evidence already identifies the account. Unbound canonical `openai` and Anthropic candidates +remain unknown during policy evaluation because Pool selection, Direct caller identity, provider +rotation, and thread affinity are resolved after the policy chooses a provider/model; a process-active +account is not used as a substitute. +Quota evidence never changes account selection, session affinity, cooldowns, or switching behavior — +it only feeds policy scoring. To see quota-aware behavior in an API dry-run, supply account refs in +the candidate evidence sent to `POST /api/routing-profiles/dry-run`: +`candidates[].codexAccountId` (Codex pool, provider `openai`) or `candidates[].accountRef` +(Anthropic) derives the matching cached account quota; an explicit `candidates[].quota` object is +echoed as given. The CLI dry-run cannot supply these per-candidate account fields. ### Combos vs policy profiles diff --git a/src/router.ts b/src/router.ts index 599ec9bb9..ec1d5bb91 100644 --- a/src/router.ts +++ b/src/router.ts @@ -22,8 +22,6 @@ import { import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec"; import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; -import { getEffectiveActiveCodexAccountId } from "./codex/routing"; -import { getAccountSet } from "./oauth/store"; import { buildRouteDecisionTrace, type RouteDecisionKind, @@ -513,24 +511,6 @@ function routeModelInternal( quota: quotaEvidenceForCandidate({ provider: candidate.provider, model: candidate.model, - ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID - && providerCodexAccountMode( - OPENAI_CODEX_PROVIDER_ID, - config.providers[OPENAI_CODEX_PROVIDER_ID], - ) === "pool" - ? (() => { - const codexAccountId = getEffectiveActiveCodexAccountId(config); - return { - codexAccountId, - codexAccountPlan: codexAccountId - ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan - : undefined, - }; - })() - : {}), - accountRef: candidate.provider === "anthropic" - ? getAccountSet("anthropic")?.activeAccountId - : undefined, }), cost: costEvidenceForCandidate({ provider: candidate.provider, diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index 36687c01b..08406b003 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -20,10 +20,6 @@ import { candidateCapabilityEvidence } from "../../routing/capability"; import { policyCandidateHealthEvidence } from "../../routing/health"; import { quotaEvidenceForCandidate } from "../../routing/quota"; import { costEvidenceForCandidate } from "../../routing/cost"; -import { providerCodexAccountMode } from "../../providers/registry"; -import { getEffectiveActiveCodexAccountId } from "../../codex/routing"; -import { getAccountSet } from "../../oauth/store"; -import { OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { saveConfigPreservingClaudeCode } from "../../config"; import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; import { isPlainRecord } from "./shared"; @@ -115,24 +111,6 @@ function assembleCandidateEvidence( quota: quotaEvidenceForCandidate({ provider: candidate.provider, model: candidate.model, - ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID - && providerCodexAccountMode( - OPENAI_CODEX_PROVIDER_ID, - config.providers[OPENAI_CODEX_PROVIDER_ID], - ) === "pool" - ? (() => { - const codexAccountId = getEffectiveActiveCodexAccountId(config); - return { - codexAccountId, - codexAccountPlan: codexAccountId - ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan - : undefined, - }; - })() - : {}), - accountRef: candidate.provider === "anthropic" - ? getAccountSet("anthropic")?.activeAccountId - : undefined, }), cost: costEvidenceForCandidate({ provider: candidate.provider, diff --git a/tests/quota-scoring.test.ts b/tests/quota-scoring.test.ts index 30a637d84..7eae3e474 100644 --- a/tests/quota-scoring.test.ts +++ b/tests/quota-scoring.test.ts @@ -7,6 +7,7 @@ import { setCachedProviderAccountQuotaForTests, clearAccountQuotaCache } from ". import { quotaEvidenceForCandidate, quotaScore } from "../src/routing/quota"; import { evaluatePolicyProfile, QUOTA_UNKNOWN_PENALTY_SCORE } from "../src/routing/evaluator"; import { routeModel } from "../src/router"; +import { getAccountSet, saveCredential } from "../src/oauth/store"; import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; import type { OcxConfig } from "../src/types"; @@ -195,7 +196,7 @@ describe("quota-aware scoring (RI-07)", () => { expect(penalized.candidates[0]!.score!.components.quota).toBe(QUOTA_UNKNOWN_PENALTY_SCORE); }); - test("execution path passes the active codex account into quota evidence", async () => { + test("execution path does not invent Codex quota evidence from the active pool account", async () => { updateAccountQuota("pool-a", 30, 1_800_000_000_000, 20, 1_900_000_000_000); const cfg = config({ codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], @@ -205,8 +206,41 @@ describe("quota-aware scoring (RI-07)", () => { }, }); const route = routeModel(cfg, "policy/quotaRoute"); - expect(route.routeDecision!.candidates[0]!.quota?.known).toBe(true); - expect(route.routeDecision!.candidates[0]!.quota?.headroom).toBeCloseTo(0.7, 2); + expect(route.routeDecision!.candidates[0]!.accountRef).toBeUndefined(); + expect(route.routeDecision!.candidates[0]!.quota?.known).toBe(false); + expect(route.routeDecision!.candidates[0]!.quota?.headroom).toBeUndefined(); + }); + + test("execution path does not invent Anthropic quota evidence from the active account", async () => { + await saveCredential("anthropic", { + access: "access-a", + refresh: "refresh-a", + expires: Date.now() + 3_600_000, + accountId: "uuid-a", + email: "a@example.test", + }); + const activeId = getAccountSet("anthropic")!.activeAccountId; + setCachedProviderAccountQuotaForTests("anthropic", activeId, { + fiveHourPercent: 40, + updatedAt: Date.now(), + }); + const cfg = config({ + providers: { + anthropic: { + adapter: "anthropic", + baseUrl: "https://api.anthropic.com", + authMode: "oauth", + models: ["claude-sonnet-5"], + }, + }, + routingProfiles: { + quotaRoute: { candidates: [{ provider: "anthropic", model: "claude-sonnet-5" }] }, + }, + }); + const route = routeModel(cfg, "policy/quotaRoute"); + expect(route.routeDecision!.candidates[0]!.accountRef).toBeUndefined(); + expect(route.routeDecision!.candidates[0]!.quota?.known).toBe(false); + expect(route.routeDecision!.candidates[0]!.quota?.headroom).toBeUndefined(); }); test("exact account selectors and pool strategies remain authoritative", () => { diff --git a/tests/routing-profile.test.ts b/tests/routing-profile.test.ts index 53aacdb8e..99dc892dd 100644 --- a/tests/routing-profile.test.ts +++ b/tests/routing-profile.test.ts @@ -476,4 +476,79 @@ describe("routing profiles (RI-04)", () => { expect(body.candidates?.[0]?.quota?.known).toBe(true); expect(body.candidates?.[0]?.quota?.headroom).toBeCloseTo(0.7, 2); }); + + test("API dry-run leaves an unbound Codex candidate quota unknown despite an active pool account", async () => { + updateAccountQuota("pool-a", 30, 1_800_000_000_000, 20, 1_900_000_000_000); + const config = baseConfig({ + providers: { + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexAccounts: [{ id: "pool-a", email: "pool-a@example.test", isMain: false }], + activeCodexAccountId: "pool-a", + routingProfiles: { + only: { candidates: [{ provider: "openai", model: "gpt-5.6" }] }, + }, + }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + // No candidates[] override: the candidate is unbound, so the dry-run must + // not reach for the process-global active pool account. Policy evaluation + // runs before Pool/Direct identity and thread affinity resolve, so an + // account attached here can differ from the one that executes. + body: JSON.stringify({ profile: "only", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { + candidates?: Array<{ accountRef?: string; quota?: { known?: boolean; headroom?: number } }>; + }; + expect(body.candidates?.[0]?.accountRef).toBeUndefined(); + expect(body.candidates?.[0]?.quota?.known).toBe(false); + expect(body.candidates?.[0]?.quota?.headroom).toBeUndefined(); + }); + + test("API dry-run leaves an unbound Anthropic candidate quota unknown despite an active account", async () => { + const { saveCredential, getAccountSet } = await import("../src/oauth/store"); + const { setCachedProviderAccountQuotaForTests } = await import("../src/providers/quota"); + await saveCredential("anthropic", { + access: "access-a", + refresh: "refresh-a", + expires: Date.now() + 3_600_000, + accountId: "uuid-a", + email: "a@example.test", + }); + const activeId = getAccountSet("anthropic")!.activeAccountId; + setCachedProviderAccountQuotaForTests("anthropic", activeId, { + fiveHourPercent: 40, + updatedAt: Date.now(), + }); + const config = baseConfig({ + providers: { + anthropic: { + adapter: "anthropic", + baseUrl: "https://api.anthropic.com", + authMode: "oauth", + models: ["claude-sonnet-5"], + }, + }, + routingProfiles: { + only: { candidates: [{ provider: "anthropic", model: "claude-sonnet-5" }] }, + }, + }); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "only", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { + candidates?: Array<{ accountRef?: string; quota?: { known?: boolean; headroom?: number } }>; + }; + expect(body.candidates?.[0]?.accountRef).toBeUndefined(); + expect(body.candidates?.[0]?.quota?.known).toBe(false); + }); }); From 14e94852517875592d4a228d09ff010ec73b55f2 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 8 Aug 2026 23:45:22 +0900 Subject: [PATCH 24/77] fix(providers): drop a removed provider's custom models (#1273) (#1293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing a provider deleted only `config.providers[name]`. Its rows in `config.customModels` stayed behind, and neither consumer filters them: `/api/models` lists every row and the generated Codex catalog emits every row keyed by slug, so the dashboard kept advertising models that resolved to a provider which no longer existed. This is an inconsistency rather than a missing feature. Provider *rename* already maintains the array — `rewriteProviderReferences` rewrites `customModels[].provider` alongside combo targets and Claude tier maps — so the array is meant to track the provider lifecycle, and one of the two sibling operations simply did not. `dropProviderCustomModels` is therefore placed next to the rename pass, so the two stay visible to each other. Both removal paths call it and report the count: `ocx provider remove` prints it and adds `droppedCustomModels` to `--json`, and the management DELETE returns the same field. An emptied list drops the `customModels` field, matching the add/remove routes; the `customModelCatalogMigration` marker is deliberately preserved, since rewriting it would change an older binary's view of one-time row ownership. Partial fix for #1273. The second defect in that report — a stale in-memory config re-persisting deleted rows through a whole-document write — is not addressed here and remains open; it needs a reconciliation design of its own, keyed on the immutable `OcxCustomModel.id` rather than on a slug. Reported by @gdxnpy with a reproduction and a before/after config diff that showed the cooperating save path itself was healthy, which is what made the second defect findable rather than "settings sometimes revert". --- src/cli/provider.ts | 7 ++ src/providers/provider-id-rewrite.ts | 29 +++++++ src/server/management/provider-routes.ts | 9 ++- tests/cli-provider.test.ts | 39 ++++++++++ tests/management-provider-validation.test.ts | 63 +++++++++++++++ tests/provider-id-rewrite.test.ts | 80 +++++++++++++++++++- 6 files changed, 225 insertions(+), 2 deletions(-) diff --git a/src/cli/provider.ts b/src/cli/provider.ts index 34eb1cd70..a418a1d58 100644 --- a/src/cli/provider.ts +++ b/src/cli/provider.ts @@ -12,6 +12,7 @@ import { apiKeyTransportConfigError, hasOwnProvider, isValidProviderName, loadCo import { hasHelpFlag } from "./help"; import { getProviderRegistryEntry, PROVIDER_REGISTRY } from "../providers/registry"; import { providerConfigSeed } from "../providers/derive"; +import { dropProviderCustomModels } from "../providers/provider-id-rewrite"; import type { OcxProviderConfig } from "../types"; import { findLiveProxy } from "../server/proxy-liveness"; import { syncModelsToCodex } from "../codex/sync"; @@ -302,6 +303,7 @@ function handleRemove(args: string[]): void { } delete config.providers[name]; + const droppedCustomModels = dropProviderCustomModels(config, name); validateAndSave(config); @@ -312,11 +314,16 @@ function handleRemove(args: string[]): void { remainingProviders: Object.keys(config.providers), defaultProvider: config.defaultProvider, needsSync: true, + ...(droppedCustomModels > 0 ? { droppedCustomModels } : {}), }, null, 2)); return; } console.log(`✅ Provider "${name}" removed.`); + if (droppedCustomModels > 0) { + const plural = droppedCustomModels === 1 ? "model" : "models"; + console.log(` Also removed ${droppedCustomModels} custom ${plural} that belonged to it.`); + } } // --------------------------------------------------------------------------- diff --git a/src/providers/provider-id-rewrite.ts b/src/providers/provider-id-rewrite.ts index 2a52399a7..f34e78b32 100644 --- a/src/providers/provider-id-rewrite.ts +++ b/src/providers/provider-id-rewrite.ts @@ -148,3 +148,32 @@ export function rewriteProviderReferences(config: OcxConfig, from: string, to: s return { changed, collisions }; } + +/** + * Drop the custom-model rows that belonged to a provider being removed. + * + * The sibling of the rename pass above. `rewriteProviderReferences` already + * carries `customModels[].provider` across a rename, so the array tracks the + * provider lifecycle — but removal used to delete only `config.providers[name]` + * and leave the rows behind. Those orphans still reach `/api/models` and the + * generated Codex catalog, which key on the row rather than on provider + * existence, so they surface as models that resolve to nothing (#1273). + * + * Only the rows are touched: the `customModelCatalogMigration` marker records + * one-time ownership of pre-marker rows and must survive removal unchanged, or + * an older binary's view of that ownership silently changes. + * + * Returns the number of rows dropped so callers can report it. + */ +export function dropProviderCustomModels(config: OcxConfig, provider: string): number { + const existing = config.customModels; + if (!Array.isArray(existing) || existing.length === 0) return 0; + const kept = existing.filter(model => model.provider !== provider); + if (kept.length === existing.length) return 0; + // Match the add/remove routes: an emptied list is dropped rather than left as + // `[]`, so the `customModels` field is absent either way. Only that field — + // the `customModelCatalogMigration` marker is deliberately left in place. + if (kept.length > 0) config.customModels = kept; + else delete config.customModels; + return existing.length - kept.length; +} diff --git a/src/server/management/provider-routes.ts b/src/server/management/provider-routes.ts index a196deabc..8a4a50d98 100644 --- a/src/server/management/provider-routes.ts +++ b/src/server/management/provider-routes.ts @@ -615,13 +615,20 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise 0 ? { droppedCustomModels } : {}), + catalogRefresh, + }); } if (url.pathname === "/api/provider-context-caps" && req.method === "GET") { diff --git a/tests/cli-provider.test.ts b/tests/cli-provider.test.ts index 20b9d42cf..e7b8e1e6a 100644 --- a/tests/cli-provider.test.ts +++ b/tests/cli-provider.test.ts @@ -223,6 +223,45 @@ describe("ocx provider", () => { } }); + test("provider remove drops that provider's custom models (#1273)", () => { + const { dir } = freshConfig({ + providers: { + openai: { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", authMode: "forward" }, + huggingface: { adapter: "openai-chat", baseUrl: "https://api.hf.test/v1", apiKey: "k" }, + }, + customModels: [ + { id: "keep-1", provider: "openai", modelId: "kept-model" }, + { id: "drop-1", provider: "huggingface", modelId: "DeepSeek-V4-Flash-0731" }, + ], + // Seeded so the assertion below proves removal does not rewrite one-time + // ownership: an older binary must keep seeing the same legacy slugs. + customModelCatalogMigration: { + version: 1, + legacyOwnedSlugs: ["huggingface/DeepSeek-V4-Flash-0731", "openai/kept-model"], + }, + }); + try { + const result = runCli(["provider", "remove", "huggingface", "--json"], { OPENCODEX_HOME: dir }); + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + action: "removed", + provider: "huggingface", + droppedCustomModels: 1, + }); + + const config = readConfig(dir); + expect(config.customModels).toEqual([ + { id: "keep-1", provider: "openai", modelId: "kept-model" }, + ]); + expect(config.customModelCatalogMigration).toEqual({ + version: 1, + legacyOwnedSlugs: ["huggingface/DeepSeek-V4-Flash-0731", "openai/kept-model"], + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("provider remove rejects default provider", () => { const { dir } = freshConfig(); try { diff --git a/tests/management-provider-validation.test.ts b/tests/management-provider-validation.test.ts index 1eb622584..81ab8133b 100644 --- a/tests/management-provider-validation.test.ts +++ b/tests/management-provider-validation.test.ts @@ -1072,6 +1072,69 @@ describe("provider management validation", () => { } }); + test("provider deletion removes that provider's custom models (#1273)", async () => { + if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + mkdirSync(TEST_DIR, { recursive: true }); + process.env.OPENCODEX_HOME = TEST_DIR; + saveConfig({ + port: 0, + defaultProvider: "test-openai", + providers: { + "test-openai": { + adapter: "openai-chat", + baseUrl: "https://api.example.test/v1", + apiKey: "sk-secret-value", + }, + removable: { + adapter: "openai-chat", + baseUrl: "https://api.removable.test/v1", + apiKey: "sk-removable", + }, + }, + customModels: [ + { id: "keep-1", provider: "test-openai", modelId: "kept-model" }, + { id: "drop-1", provider: "removable", modelId: "ghost-model" }, + ], + // Seeded so the assertion below covers the real persistence path, not just + // the helper: `projectCustomModelCatalogMigration` runs inside the save and + // must carry this marker through a provider delete unchanged. + customModelCatalogMigration: { + version: 1, + legacyOwnedSlugs: ["removable/ghost-model", "test-openai/kept-model"], + }, + } as unknown as Parameters[0]); + + const server = startServer(0); + try { + const response = await fetch(new URL("/api/providers?name=removable", server.url), { + method: "DELETE", + }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ success: true, droppedCustomModels: 1 }); + + // The dashboard model page reads this route; a surviving row here is the + // ghost model users see pointing at a provider that no longer exists. + const customModels = await fetch(new URL("/api/custom-models", server.url)); + expect(await customModels.json()).toEqual([ + { id: "keep-1", provider: "test-openai", modelId: "kept-model" }, + ]); + + const persisted = JSON.parse(readFileSync(join(TEST_DIR, "config.json"), "utf8")) as { + customModels?: unknown; + customModelCatalogMigration?: unknown; + }; + expect(persisted.customModels).toEqual([ + { id: "keep-1", provider: "test-openai", modelId: "kept-model" }, + ]); + expect(persisted.customModelCatalogMigration).toEqual({ + version: 1, + legacyOwnedSlugs: ["removable/ghost-model", "test-openai/kept-model"], + }); + } finally { + await server.stop(true); + } + }); + test("provider management switches the default and reassigns it when removed", async () => { if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); mkdirSync(TEST_DIR, { recursive: true }); diff --git a/tests/provider-id-rewrite.test.ts b/tests/provider-id-rewrite.test.ts index 693653b0f..5c6fc5a8b 100644 --- a/tests/provider-id-rewrite.test.ts +++ b/tests/provider-id-rewrite.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; import { comboConfigError } from "../src/combos"; import { providerContextCap } from "../src/providers/context-cap"; -import { rewriteProviderReferences } from "../src/providers/provider-id-rewrite"; +import { dropProviderCustomModels, rewriteProviderReferences } from "../src/providers/provider-id-rewrite"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; const FROM = "alibaba-token-plan"; @@ -131,3 +131,81 @@ test("does not touch providers[*].selectedModels", () => { expect(rewriteProviderReferences(config, FROM, TO).changed).toBe(0); expect(config.providers.openrouter!.selectedModels).toEqual([`${FROM}/qwen3.7-max`]); }); + +// --------------------------------------------------------------------------- +// dropProviderCustomModels — the removal sibling of the rename pass (#1273) +// --------------------------------------------------------------------------- + +function customModelsConfig(models: Array<{ id: string; provider: string; modelId: string }>): OcxConfig { + return { + providers: { + huggingface: { adapter: "openai-chat" }, + "agnes-ai": { adapter: "openai-chat" }, + }, + customModels: models, + } as unknown as OcxConfig; +} + +test("removal drops only the departing provider's custom models", () => { + const config = customModelsConfig([ + { id: "a", provider: "agnes-ai", modelId: "agnes-2.5-flash" }, + { id: "b", provider: "huggingface", modelId: "DeepSeek-V4-Flash-0731" }, + { id: "c", provider: "huggingface", modelId: "another-model" }, + ]); + + expect(dropProviderCustomModels(config, "huggingface")).toBe(2); + expect(config.customModels).toEqual([ + { id: "a", provider: "agnes-ai", modelId: "agnes-2.5-flash" }, + ] as OcxConfig["customModels"]); +}); + +test("removing the last custom model deletes the key rather than leaving []", () => { + // The add/remove routes drop an emptied list, so the `customModels` field is + // absent either way. Only that field: the `customModelCatalogMigration` + // marker is deliberately preserved, so the two configs are not identical. + const config = customModelsConfig([ + { id: "b", provider: "huggingface", modelId: "DeepSeek-V4-Flash-0731" }, + ]); + + expect(dropProviderCustomModels(config, "huggingface")).toBe(1); + expect(Object.hasOwn(config, "customModels")).toBe(false); +}); + +test("a provider with no custom models is a no-op that leaves the array identical", () => { + const rows = [{ id: "a", provider: "agnes-ai", modelId: "agnes-2.5-flash" }]; + const config = customModelsConfig(rows); + const before = config.customModels; + + expect(dropProviderCustomModels(config, "huggingface")).toBe(0); + // Same reference, not merely a deep-equal copy: an untouched save must not + // look like a mutation to anything comparing identity. + expect(config.customModels).toBe(before); +}); + +test("an absent customModels key is left absent", () => { + const config = { providers: { huggingface: { adapter: "openai-chat" } } } as unknown as OcxConfig; + expect(dropProviderCustomModels(config, "huggingface")).toBe(0); + expect(Object.hasOwn(config, "customModels")).toBe(false); +}); + +test("removal leaves the custom-model ownership marker untouched", () => { + // legacyOwnedSlugs records one-time ownership of pre-marker rows. Rewriting it + // here would change an older binary's view of what it may delete, which the + // migration module explicitly warns against. + const config = customModelsConfig([ + { id: "a", provider: "agnes-ai", modelId: "agnes-2.5-flash" }, + { id: "b", provider: "huggingface", modelId: "DeepSeek-V4-Flash-0731" }, + ]); + const marker = { + version: 1, + legacyOwnedSlugs: ["agnes-ai/agnes-2.5-flash", "huggingface/DeepSeek-V4-Flash-0731"], + }; + (config as unknown as Record).customModelCatalogMigration = marker; + + dropProviderCustomModels(config, "huggingface"); + + expect((config as unknown as Record).customModelCatalogMigration).toEqual({ + version: 1, + legacyOwnedSlugs: ["agnes-ai/agnes-2.5-flash", "huggingface/DeepSeek-V4-Flash-0731"], + }); +}); From 794d8eb094fc7b0b4ac30e0accfbf5bf11aa3c33 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 9 Aug 2026 01:33:47 +0900 Subject: [PATCH 25/77] fix(catalog): synthesize incomplete combo members with context fallback (#1305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A combo member whose provider row is incomplete — missing a context window, or absent from the fetched catalog — was dropped from the generated Codex catalog rather than synthesized. `resolveComboCatalogMember` fills the gap from configuration, falling back to the provider's declared max input and then to 128k, so a combo stays selectable when one member's upstream row is thin. Republished from #1163 by 关俊江, whose branch was 366 commits behind dev. Two conflicts, both mechanical and both on a single line: `dev` renamed `augmentRoutedModelsWithJawcodeMetadata` to `augmentRoutedModelsWithMetadata` and added `CODEX_ACCOUNT_BOUND_CATALOG_KIND` plus a `catalog/parsing` import block, while this branch added `resolveComboCatalogMember` to the same export and import lines. Resolved by keeping every symbol from both sides; no behavior was re-decided. Co-authored-by: 关俊江 --- .../docs/reference/configuration/routing.md | 13 +- .../zh-cn/reference/configuration/routing.md | 7 +- src/codex/catalog.ts | 2 +- src/codex/catalog/aggregation.ts | 11 +- src/codex/catalog/provider-fetch.ts | 111 +++++++- tests/codex-catalog.test.ts | 250 +++++++++++++++++- 6 files changed, 375 insertions(+), 19 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index d51e152ad..333422f06 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -184,13 +184,16 @@ Per-request route-decision traces are recorded when a policy profile executes. A combo remains directly routable even when it cannot be listed. `ocx sync`, `/v1/models`, and the Codex picker list it only when every target exposes capabilities that can be intersected: -- a positive `contextWindow`, from live metadata, registry hints, or provider - `modelContextWindows` / `contextWindow`; and +- a positive `contextWindow`, from live metadata, registry hints, provider + `modelContextWindows` / `contextWindow`, a known positive `maxInputTokens` on the member row, + or — when the provider is known and enabled but every source still omits a window — a + conservative 128,000-token fallback (clamped by `providerContextCaps` when set); and - a non-empty `inputModalities` intersection, treating an omitted member value as `["text"]`. -A bare relay id with no context metadata or targets with disjoint modalities removes the combo from -the catalog. Sync emits a summary warning and the dashboard marks it **Needs attention**. Add context -metadata, align modalities, or target models with discoverable compatible capabilities. +A target on a disabled provider (even with a complete discovery row), on an unknown provider with +no discovery row, or targets with disjoint modalities, removes the combo from the catalog. Sync +emits a summary warning and the dashboard marks it **Needs attention**. Add context metadata, +align modalities, or target models with discoverable compatible capabilities. ## Request history and routing analytics diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md index b718ed4ed..18d08cbe1 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md @@ -82,11 +82,12 @@ selector 校验、冲突规则和隐私说明见[提供方配置](/reference/con 即使某个 combo 不能被列出，它仍然可以直接路由。只有当所有目标都暴露出可以交集的能力时，`ocx sync`、`/v1/models` 和 Codex 选择器才会列出它： -- 一个正的 `contextWindow`，来源可以是实时元数据、注册表提示，或提供方的 - `modelContextWindows` / `contextWindow`；以及 +- 一个正的 `contextWindow`，来源可以是实时元数据、注册表提示、提供方的 + `modelContextWindows` / `contextWindow`、成员行上已知的正 `maxInputTokens`，或者——当提供方已知且启用但所有来源仍未给出窗口时—— + 保守的 128,000 token 回退（若配置了 `providerContextCaps` 则会按上限夹紧）；以及 - 非空的 `inputModalities` 交集，其中省略的成员值按 `["text"]` 处理。 -如果是一个没有上下文元数据的裸 relay id，或者目标之间的模态互不相交，combo 就会从 +目标位于已禁用提供方（即使有完整 discovery 行）、未知且无 discovery 行的提供方，或目标之间的模态互不相交时，combo 会从 目录中移除。同步时会输出一条汇总警告，仪表板会将其标记为 **Needs attention**。 补充上下文元数据、对齐模态，或者把目标模型切换为可发现且兼容的能力。 diff --git a/src/codex/catalog.ts b/src/codex/catalog.ts index 8a98f0ff7..1ba42c1e7 100644 --- a/src/codex/catalog.ts +++ b/src/codex/catalog.ts @@ -5,7 +5,7 @@ export type { CatalogModel, MultiAgentMode } from "./catalog/parsing"; export { NATIVE_OPENAI_MODELS, nativeOpenAiContextWindow, disabledNativeSlugs, visibleNativeSlugs, desktopVisibleNativeSlugs, nativeModelRows, applyNativeVisibility, upstreamNativeEntry, nativeOpenAiSlugs, listCatalogNativeSlugs, nativeReasoningEfforts, nativeDefaultReasoningEffort, shouldIncludeAccountBoundNativeOpenAi, shouldIncludeNativeOpenAi } from "./catalog/metadata"; export { isSpawnableCodexCandidate, codexExecInvocation, loadBundledCodexCatalog, materializeBundledCodexCatalog, loadCatalogTemplate } from "./catalog/bundled"; export { nativeEffortClamp, shouldApplyNativeEffortClamp, catalogModelEfforts, codexSupportedReasoningEfforts, clampedDefaultEffort, clampEntryToCodexSupportedEfforts, clampCatalogModelsToCodexSupport } from "./catalog/effort"; -export { applyProviderConfigHints, isDatedVariantId, filterCatalogVisibleModels, gatherRoutedModels, clearGatherRoutedModelsInflight, augmentRoutedModelsWithRegistryOpenAiApiRows, augmentRoutedModelsWithMetadata } from "./catalog/provider-fetch"; +export { applyProviderConfigHints, isDatedVariantId, filterCatalogVisibleModels, gatherRoutedModels, clearGatherRoutedModelsInflight, augmentRoutedModelsWithRegistryOpenAiApiRows, augmentRoutedModelsWithMetadata, resolveComboCatalogMember } from "./catalog/provider-fetch"; export { deriveComboCatalogModel, exactComboCatalogSlugs, getLastComboCatalogOmissions, resetOpenAiApiCatalogWarningStateForTests, uniqueCatalogModelsForPublicList, uniqueCatalogModelsForRawPublicList, buildComboCatalogOmission, comboCatalogOmissionReason, summarizeComboCatalogOmissions } from "./catalog/aggregation"; export type { ComboCatalogOmission, ComboCatalogOmissionReason } from "./catalog/aggregation"; export { MAX_SPAWN_AGENT_MODEL_OVERRIDES, CANONICAL_NATIVE_CATALOG_CONTENT_POLICY, effectiveSubagentRoster, buildCatalogEntries, mergeCatalogEntriesFromObservedState, resetCatalogRuntimeStateForTests, orderForSubagents, mergeCatalogEntriesForSync, syncCatalogModels, restoreCodexCatalog, invalidateCodexModelsCache } from "./catalog/sync"; diff --git a/src/codex/catalog/aggregation.ts b/src/codex/catalog/aggregation.ts index cb1d9fd07..89ee7feaa 100644 --- a/src/codex/catalog/aggregation.ts +++ b/src/codex/catalog/aggregation.ts @@ -131,9 +131,14 @@ export function deriveComboCatalogModel( const inputModalities = intersectStrings( members.map(member => member.inputModalities ?? ["text"]), ); - const reasoningEfforts = intersectStrings( - members.map(member => member.reasoningEfforts ?? []), - ); + // Unknown ladders (`undefined`) are wildcards for catalog derivation — same + // boundary as the GUI picker. An explicit empty ladder still constrains. + const advertisedLadders = members + .map(member => member.reasoningEfforts) + .filter((ladder): ladder is string[] => ladder !== undefined); + const reasoningEfforts = advertisedLadders.length === 0 + ? [] + : intersectStrings(advertisedLadders); const contextWindow = Math.min(...members.map(member => member.contextWindow!)); const maxInputTokens = Math.min( ...members.map(member => member.maxInputTokens ?? member.contextWindow!), diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index 79d3d55b9..a92b8a539 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -620,6 +620,104 @@ export function applyConfigHintsToCachedModels(name: string, prov: OcxProviderCo return models.map(model => applyProviderConfigHints(name, prov, model, contextCap)); } + +/** + * Last-resort context window for combo member synthesis when discovery and + * provider config both omit one. Matches the catalog entry default in + * `normalizeRoutedCatalogEntry` so incomplete live rows still catalog. + */ +const COMBO_MEMBER_CONTEXT_FALLBACK = 128_000; + +/** + * Resolve a combo target to a catalog member for derivation. + * Prefer discovery metadata; when the target is missing from the gather map or + * lacks a positive contextWindow, synthesize from the (registry-enriched) + * provider config so combos remain catalogued when targets are configured but + * discovery metadata is incomplete. Disabled providers stay unresolved. + * When hints still omit contextWindow, prefer known maxInputTokens, else + * COMBO_MEMBER_CONTEXT_FALLBACK so a live row without ctx does not drop the + * whole combo from the public catalog. + */ +export function resolveComboCatalogMember( + target: { provider: string; model: string }, + memberByKey: ReadonlyMap, + providers: ReadonlyMap, + contextCap?: number, +): CatalogModel | undefined { + const existing = memberByKey.get(targetKey(target)); + const prov = providers.get(target.provider); + // Disabled providers never contribute members — even a complete discovery row + // is unusable for catalog derivation while the provider is off. + if (prov?.disabled === true) return undefined; + + // Complete live/configured rows still honor providerContextCaps so a high + // discovery window cannot outrun an operator-configured cap. + if ( + existing + && typeof existing.contextWindow === "number" + && existing.contextWindow > 0 + ) { + const capped = applyProviderContextCap(existing.contextWindow, contextCap); + if (capped === undefined || capped === existing.contextWindow) return existing; + const maxInput = typeof existing.maxInputTokens === "number" && existing.maxInputTokens > 0 + ? Math.min(existing.maxInputTokens, capped) + : capped; + return { + ...existing, + contextWindow: capped, + maxInputTokens: maxInput, + contextCap, + contextCapped: true as const, + }; + } + + const base: CatalogModel = existing ?? { + id: target.model, + provider: target.provider, + }; + const hinted = prov + ? applyProviderConfigHints(target.provider, prov, base, contextCap) + : base; + const hintedContext = typeof hinted.contextWindow === "number" && hinted.contextWindow > 0 + ? hinted.contextWindow + : undefined; + // Prefer a known positive maxInputTokens over inventing 128k when discovery + // advertised an input limit but no context window (common thin /models rows). + const knownMaxInput = typeof hinted.maxInputTokens === "number" && hinted.maxInputTokens > 0 + ? hinted.maxInputTokens + : (typeof base.maxInputTokens === "number" && base.maxInputTokens > 0 + ? base.maxInputTokens + : undefined); + const uncappedContext = hintedContext + ?? knownMaxInput + ?? (existing || prov ? COMBO_MEMBER_CONTEXT_FALLBACK : undefined); + if (uncappedContext === undefined) return undefined; + const usedFallback = hintedContext === undefined; + const cappedContext = applyProviderContextCap(uncappedContext, contextCap); + const contextWindow = cappedContext ?? uncappedContext; + const fallbackCapped = usedFallback + && contextCap !== undefined + && cappedContext !== undefined + && cappedContext !== uncappedContext; + + const inputModalities = hinted.inputModalities ?? base.inputModalities ?? ["text"]; + const reasoningEfforts = hinted.reasoningEfforts + ?? (prov ? configuredReasoningEfforts(prov, target.model) : undefined) + ?? base.reasoningEfforts; + const maxInputTokens = knownMaxInput !== undefined + ? Math.min(knownMaxInput, contextWindow) + : contextWindow; + + return { + ...hinted, + inputModalities, + ...(reasoningEfforts !== undefined ? { reasoningEfforts } : {}), + contextWindow, + maxInputTokens, + ...(fallbackCapped ? { contextCap, contextCapped: true as const } : {}), + }; +} + export function isDatedVariantId(liveId: string, configuredId: string): boolean { if (!liveId.startsWith(`${configuredId}-`)) return false; return /^\d{8}$/.test(liveId.slice(configuredId.length + 1)); @@ -1319,11 +1417,19 @@ async function gatherRoutedModelsUncached( if (!memberByKey.has(key)) memberByKey.set(key, synthetic); } } + // Enriched (registry-hydrated) provider clones — shared by combo member synthesis and + // custom-model vision-sidecar inheritance so both see the same merged registry view. + const enrichedByName = new Map(activeProviders.map(provider => [provider.name, provider.provider])); for (const id of listComboIds(config)) { const combo = getCombo(config, id); if (!combo) continue; const members = combo.targets - .map(target => memberByKey.get(targetKey(target))) + .map(target => resolveComboCatalogMember( + target, + memberByKey, + enrichedByName, + providerContextCap(config, target.provider), + )) .filter((member): member is CatalogModel => member !== undefined); const derived = deriveComboCatalogModel(id, combo, members); if (derived) all.push(derived); @@ -1331,9 +1437,6 @@ async function gatherRoutedModelsUncached( } replaceLastComboCatalogOmissions(localOmissions); all.sort((a, b) => (a.provider === b.provider ? a.id.localeCompare(b.id) : a.provider.localeCompare(b.provider))); - // Enriched (registry-hydrated) provider clones, keyed by name — the same view used above so - // custom rows get the same noVisionModels / inputModalities treatment as discovered rows. - const enrichedByName = new Map(activeProviders.map(provider => [provider.name, provider.provider])); // Provider-derived rows keyed by their Codex-facing slug: a custom override replaces the row // with the same slug below, so that row's provider capability metadata is the inheritance source. const replacedByRoutedSlug = new Map(all.map(model => [routedSlug(model.provider, model.id), model])); diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index a46835463..f3c01d199 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { augmentRoutedModelsWithMetadata, augmentRoutedModelsWithRegistryOpenAiApiRows, buildCatalogEntries, buildComboCatalogOmission, catalogModelSlug, clampCatalogModelsToCodexSupport, clampEntryToCodexSupportedEfforts, clampedDefaultEffort, CODEX_ACCOUNT_BOUND_CATALOG_KIND, comboCatalogOmissionReason, deriveComboCatalogModel, exactComboCatalogSlugs, filterCatalogVisibleModels, filterSupportedNativeSlugs, gatherRoutedModels as gatherRoutedModelsDirect, isDatedVariantId, isMediaGenerationModelId, loadBundledCodexCatalog, materializeBundledCodexCatalog, mergeCatalogEntriesForSync, NATIVE_OPENAI_MODELS, normalizeRoutedCatalogEntry, resetCatalogRuntimeStateForTests, resetOpenAiApiCatalogWarningStateForTests, shouldExposeRoutedModel } from "../src/codex/catalog"; +import { augmentRoutedModelsWithMetadata, augmentRoutedModelsWithRegistryOpenAiApiRows, buildCatalogEntries, buildComboCatalogOmission, catalogModelSlug, clampCatalogModelsToCodexSupport, clampEntryToCodexSupportedEfforts, clampedDefaultEffort, CODEX_ACCOUNT_BOUND_CATALOG_KIND, comboCatalogOmissionReason, deriveComboCatalogModel, exactComboCatalogSlugs, filterCatalogVisibleModels, filterSupportedNativeSlugs, gatherRoutedModels as gatherRoutedModelsDirect, isDatedVariantId, isMediaGenerationModelId, loadBundledCodexCatalog, materializeBundledCodexCatalog, mergeCatalogEntriesForSync, NATIVE_OPENAI_MODELS, normalizeRoutedCatalogEntry, resetCatalogRuntimeStateForTests, resetOpenAiApiCatalogWarningStateForTests, resolveComboCatalogMember, shouldExposeRoutedModel } from "../src/codex/catalog"; import { CODEX_CUSTOM_MODEL_CATALOG_KIND, CODEX_PROVIDER_MODEL_CATALOG_KIND, @@ -65,7 +65,7 @@ function normalizedCombo( strategy: "failover", stickyLimit: 1, defaultEffort: "medium", - alias: null, + alias: null, targets: [ { provider: "a", model: "m1", weight: 1 }, { provider: "b", model: "m2", weight: 1 }, @@ -212,6 +212,24 @@ describe("combo catalog capability intersection", () => { ])?.defaultReasoningEffort).toBe("medium"); }); + test("treats undefined effort ladders as wildcards and empty ladders as restrictive", () => { + // One recovered target with no ladder must not zero the advertised intersection. + expect(deriveComboCatalogModel("wildcard", normalizedCombo({ defaultEffort: "medium" }), [ + memberA, + { ...memberB, reasoningEfforts: undefined }, + ])).toEqual(expect.objectContaining({ + reasoningEfforts: ["low", "medium", "high"], + defaultReasoningEffort: "medium", + })); + // Explicit empty ladder still constrains to nothing. + const empty = deriveComboCatalogModel("empty", normalizedCombo({ defaultEffort: "medium" }), [ + memberA, + { ...memberB, reasoningEfforts: [] }, + ]); + expect(empty?.reasoningEfforts).toEqual([]); + expect(empty).not.toHaveProperty("defaultReasoningEffort"); + }); + test("fails closed for missing members, unknown context, duplicate targets, and empty modalities", () => { expect(deriveComboCatalogModel("missing", normalizedCombo(), [memberA])).toBeNull(); expect(deriveComboCatalogModel("context", normalizedCombo(), [ @@ -892,7 +910,9 @@ describe("combo catalog capability intersection", () => { }, combos: { mixed: { targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] }, - hidden: { targets: [{ provider: "a", model: warningSentinel }] }, + // Unknown provider (not just unlisted model) — synthesis cannot invent a member, + // and the secret in the model id must still be redacted in the omission warning. + hidden: { targets: [{ provider: warningSentinel, model: "m1" }] }, }, disabledModels: ["combo/mixed"], }; @@ -1064,6 +1084,230 @@ describe("combo catalog capability intersection", () => { expect(openaiRows).toEqual([]); expect(rows.some(r => r.provider === "combo" && r.id === "solo")).toBe(true); }); + + test("synthesizes missing combo targets from provider config metadata", async () => { + // Target model is not in models[] (so never lands in memberByKey) but provider + // config carries context/modalities/efforts — combo derivation must still catalog. + const config: OcxConfig = { + port: 10100, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://a.example/v1", + liveModels: false, + models: ["listed"], + modelContextWindows: { listed: 200_000, unlisted: 128_000 }, + modelInputModalities: { unlisted: ["text"] }, + modelReasoningEfforts: { unlisted: ["low", "medium", "high"] }, + }, + b: { + adapter: "openai-chat", + baseUrl: "https://b.example/v1", + liveModels: false, + models: ["m2"], + modelContextWindows: { m2: 100_000 }, + modelReasoningEfforts: { m2: ["low", "medium"] }, + }, + }, + combos: { + recovered: { + targets: [ + { provider: "a", model: "unlisted" }, + { provider: "b", model: "m2" }, + ], + }, + }, + }; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + resetCatalogRuntimeStateForTests(); + const rows = await gatherRoutedModels(config); + const combo = rows.find(r => r.provider === "combo" && r.id === "recovered"); + expect(combo).toBeDefined(); + expect(combo!.contextWindow).toBe(100_000); + expect(combo!.inputModalities).toEqual(["text"]); + expect(combo!.reasoningEfforts).toEqual(["low", "medium"]); + // Synthesized member must not leak as a standalone routed row. + expect(rows.some(r => r.provider === "a" && r.id === "unlisted")).toBe(false); + const { getLastComboCatalogOmissions } = await import("../src/codex/catalog"); + expect(getLastComboCatalogOmissions().some(item => item.id === "recovered")).toBe(false); + } finally { + warn.mockRestore(); + } + }, 15_000); + + test("resolveComboCatalogMember fills incomplete members from provider config", () => { + // Member is present but lacks contextWindow; provider.contextWindow + efforts complete it. + const incomplete = { + provider: "a", + id: "m1", + // no contextWindow — the incomplete_metadata trigger + }; + const memberByKey = new Map([["a/m1", incomplete]]); + const providers = new Map([ + ["a", { + adapter: "openai-chat" as const, + baseUrl: "https://a.example/v1", + contextWindow: 200_000, + modelReasoningEfforts: { m1: ["low", "medium", "high"] }, + }], + ]); + const resolved = resolveComboCatalogMember( + { provider: "a", model: "m1" }, + memberByKey, + providers, + ); + expect(resolved).toMatchObject({ + provider: "a", + id: "m1", + contextWindow: 200_000, + maxInputTokens: 200_000, + inputModalities: ["text"], + reasoningEfforts: ["low", "medium", "high"], + }); + // Complete members are returned as-is without re-synthesis side effects. + const complete = { + provider: "a", + id: "m1", + contextWindow: 99_000, + inputModalities: ["text", "image"], + reasoningEfforts: ["high"], + }; + expect(resolveComboCatalogMember( + { provider: "a", model: "m1" }, + new Map([["a/m1", complete]]), + providers, + )).toBe(complete); + // Complete members still honor an operator-configured context cap. + expect(resolveComboCatalogMember( + { provider: "a", model: "m1" }, + new Map([["a/m1", complete]]), + providers, + 50_000, + )).toMatchObject({ + contextWindow: 50_000, + maxInputTokens: 50_000, + contextCap: 50_000, + contextCapped: true, + }); + // Disabled providers never contribute — even with a complete discovery row. + expect(resolveComboCatalogMember( + { provider: "a", model: "m1" }, + new Map([["a/m1", complete]]), + new Map([["a", { adapter: "openai-chat", baseUrl: "https://a.example/v1", disabled: true }]]), + )).toBeUndefined(); + // Thin live rows with max_input_tokens but no contextWindow prefer that limit + // over inventing the 128k fallback. + expect(resolveComboCatalogMember( + { provider: "a", model: "thin" }, + new Map([["a/thin", { provider: "a", id: "thin", maxInputTokens: 8_192 }]]), + new Map([["a", { adapter: "openai-chat", baseUrl: "https://a.example/v1" }]]), + )).toMatchObject({ + contextWindow: 8_192, + maxInputTokens: 8_192, + }); + // Known provider without context metadata still gets the conservative fallback. + expect(resolveComboCatalogMember( + { provider: "a", model: "ghost" }, + new Map(), + new Map([["a", { adapter: "openai-chat", baseUrl: "https://a.example/v1" }]]), + )).toMatchObject({ + provider: "a", + id: "ghost", + contextWindow: 128_000, + maxInputTokens: 128_000, + inputModalities: ["text"], + }); + // Provider contextCap below the 128k fallback clamps the synthesized window. + expect(resolveComboCatalogMember( + { provider: "a", model: "ghost" }, + new Map(), + new Map([["a", { adapter: "openai-chat", baseUrl: "https://a.example/v1" }]]), + 64_000, + )).toMatchObject({ + provider: "a", + id: "ghost", + contextWindow: 64_000, + maxInputTokens: 64_000, + contextCap: 64_000, + contextCapped: true, + }); + // Cap above the fallback leaves 128k (no artificial raise, no capped flag). + const aboveCap = resolveComboCatalogMember( + { provider: "a", model: "ghost" }, + new Map(), + new Map([["a", { adapter: "openai-chat", baseUrl: "https://a.example/v1" }]]), + 200_000, + ); + expect(aboveCap).toMatchObject({ + contextWindow: 128_000, + maxInputTokens: 128_000, + }); + // Cap may be recorded for bookkeeping (contextCapped: false) but must not claim a clamp. + expect(aboveCap?.contextCapped).toBeFalsy(); + // No provider entry and no discovery row — cannot invent a member. + expect(resolveComboCatalogMember( + { provider: "missing", model: "ghost" }, + new Map(), + new Map(), + )).toBeUndefined(); + }); + + test("still omits combos when synthesis cannot recover hard failures", async () => { + const config: OcxConfig = { + port: 10100, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://a.example/v1", + liveModels: false, + models: ["m1"], + modelContextWindows: { m1: 128_000 }, + // Disjoint modalities with b → empty intersection (incompatible_modalities). + modelInputModalities: { m1: ["image"] }, + }, + b: { + adapter: "openai-chat", + baseUrl: "https://b.example/v1", + liveModels: false, + models: ["m2"], + modelContextWindows: { m2: 128_000 }, + modelInputModalities: { m2: ["audio"] }, + }, + }, + combos: { + disjoint: { + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + ghost: { + // Provider does not exist — synthesis cannot invent a member. + targets: [{ provider: "missing-provider", model: "never-configured" }], + }, + }, + }; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + resetCatalogRuntimeStateForTests(); + const rows = await gatherRoutedModels(config); + expect(rows.some(r => r.provider === "combo" && r.id === "disjoint")).toBe(false); + expect(rows.some(r => r.provider === "combo" && r.id === "ghost")).toBe(false); + const { getLastComboCatalogOmissions } = await import("../src/codex/catalog"); + const omissions = getLastComboCatalogOmissions(); + expect(omissions.find(item => item.id === "disjoint")).toMatchObject({ + reason: "incompatible_modalities", + }); + expect(omissions.find(item => item.id === "ghost")).toMatchObject({ + reason: "incomplete_metadata", + }); + } finally { + warn.mockRestore(); + } + }, 15_000); }); describe("Google Gemini catalog metadata", () => { From e8ec8d1918a034b3ec7e2266d1773cd1e4ef05c3 Mon Sep 17 00:00:00 2001 From: iF2007 Date: Sun, 9 Aug 2026 00:33:50 +0800 Subject: [PATCH 26/77] fix(google): discover Antigravity live models (#1178) * fix(google): discover Antigravity live models * fix(security): guard Antigravity model discovery Route Cloud Code Assist discovery POSTs through the provider outbound policy and invalidate account-scoped live model cache when OAuth credentials change. * fix(google): scope Antigravity discovery to current account Discard stale discovery writes after OAuth account changes, use the routed CCA project for discovery, and retain partial Gemini availability as explicit wire IDs. * fix(google): harden Antigravity catalog discovery Invalidate removed provider generations, migrate canonical static opt-outs, and reject malformed CCA model identifiers. * fix(antigravity): harden live model discovery * fix(catalog): distinguish cache eviction from authority changes --- docs-site/src/content/docs/contributing.md | 3 +- .../src/content/docs/guides/providers.md | 2 +- .../src/content/docs/ja/guides/providers.md | 2 +- .../src/content/docs/ko/guides/providers.md | 2 +- .../src/content/docs/ru/guides/providers.md | 2 +- .../content/docs/zh-cn/guides/providers.md | 2 +- src/codex/catalog/provider-fetch.ts | 92 +++++++-- src/codex/catalog/sync.ts | 2 +- src/codex/convergence-types.ts | 2 +- src/codex/model-cache.ts | 60 +++++- src/config.ts | 9 +- src/lib/pinned-http.ts | 49 ++++- src/lib/provider-outbound.ts | 51 ++++- src/oauth/index.ts | 102 ++++++---- src/providers/antigravity-models.ts | 118 ++++++++++- src/providers/model-discovery-limits.ts | 16 ++ src/providers/model-discovery.ts | 29 ++- src/providers/registry.ts | 4 +- src/server/management/oauth-account-routes.ts | 12 ++ src/server/management/provider-routes.ts | 51 +++-- src/server/responses/core.ts | 6 +- src/types.ts | 4 +- tests/antigravity-static-catalog.test.ts | 1 + tests/codex-catalog.test.ts | 3 +- tests/config.test.ts | 8 +- tests/cursor-hardening.test.ts | 38 ++++ tests/google-antigravity-wire.test.ts | 78 +++++++- tests/google-models-listing.test.ts | 189 +++++++++++++++++- tests/helpers/provider-registry-discovery.ts | 4 +- tests/model-cache.test.ts | 46 +++++ tests/oauth-accounts-api.test.ts | 104 ++++++++++ tests/oauth-provider-reconcile.test.ts | 50 +++-- tests/provider-connection-test.test.ts | 42 +++- tests/provider-outbound.test.ts | 79 +++++++- tests/provider-registry-parity.test.ts | 6 +- 35 files changed, 1087 insertions(+), 181 deletions(-) create mode 100644 src/providers/model-discovery-limits.ts create mode 100644 tests/model-cache.test.ts diff --git a/docs-site/src/content/docs/contributing.md b/docs-site/src/content/docs/contributing.md index f8954b90e..bf8017e5b 100644 --- a/docs-site/src/content/docs/contributing.md +++ b/docs-site/src/content/docs/contributing.md @@ -164,7 +164,8 @@ sent to. A preset therefore needs primary-source evidence, not a working code pa that add or promote a provider must supply all of the following in the description: - **The documented OpenAI-compatible endpoints.** Link the vendor's own API reference for the chat - endpoint and, when the entry sets `liveModels: true`, for authenticated `GET /v1/models`. A + endpoint and, when the entry sets `liveModels: true`, for its authenticated model-discovery + endpoint (typically `GET /v1/models`). A passing fixture test is not a substitute: it proves our code shape, not the upstream contract. - **Terms of service and the operating legal entity.** An empty or placeholder legal page does not establish who runs the endpoint or under what terms user traffic is handled. diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index af1d0ef4a..07d0677e1 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -113,7 +113,7 @@ ocx logout | `anthropic` | `anthropic` | `https://api.anthropic.com` | Claude models; live model list fetched from `/v1/models`. | | `kimi` | `openai-chat` | `https://api.kimi.com/coding/v1` | Kimi K2.7/K2.6/K2.5 coding models. | | `kiro` | `kiro` | `https://runtime.us-east-1.kiro.dev` | Initial login imports the installed, signed-in `kiro-cli` session (on Unix, install with `curl -fsSL https://cli.kiro.dev/install | bash`; on Windows PowerShell, use `irm 'https://cli.kiro.dev/install.ps1' | iex`; then run `kiro-cli login`). **Add account** logs `kiro-cli` out, starts a fresh browser login that switches the account used by `kiro-cli`, and stores account-scoped profile metadata. Existing OpenCodex accounts are preserved, and cancellation or failure restores the previous `kiro-cli` session. | -| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth over the Cloud Code Assist wire. Uses the maintained six-model static catalog because CCA does not expose the generic `/models` endpoint. | +| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth over the Cloud Code Assist wire. Live discovery uses CCA's authenticated `v1internal:fetchAvailableModels` endpoint and publishes the agent models available to the signed-in account; the maintained catalog remains the fallback. | | `cursor` | `cursor` | `https://api2.cursor.sh` | Experimental PKCE login, live HTTP/2 transport, and account-filtered model discovery. | | `github-copilot` | `openai-chat` | `https://api.githubcopilot.com` | Experimental. GitHub device flow + `copilot_internal` exchange (VS Code OAuth client). Requires an active Copilot subscription; not an official third-party API. | diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index a5d15561b..da7e63317 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -108,7 +108,7 @@ ocx logout | `anthropic` | `anthropic` | `https://api.anthropic.com` | Claude モデル; ライブモデル一覧は `/v1/models` から取得。 | | `kimi` | `openai-chat` | `https://api.kimi.com/coding/v1` | Kimi K2.7/K2.6/K2.5 コーディングモデル。 | | `kiro` | `kiro` | `https://runtime.us-east-1.kiro.dev` | 初回ログインは、インストール済みでサインインした `kiro-cli` セッションを取り込みます（Unix では `curl -fsSL https://cli.kiro.dev/install | bash`、Windows PowerShell では `irm 'https://cli.kiro.dev/install.ps1' | iex` でインストールしてから `kiro-cli login` を実行）。**アカウントを追加**は `kiro-cli` をログアウトして新しいブラウザログインを開始し、`kiro-cli` 自体のアカウントを切り替えてアカウント別プロファイルメタデータを保存します。既存の OpenCodex アカウントは保持され、キャンセルまたは失敗時には以前の `kiro-cli` セッションが復元されます。 | -| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth を Cloud Code Assist wire で使用。CCA は汎用 `/models` エンドポイントを公開しないため、管理された 6 モデルの静的カタログを使用します。 | +| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth を Cloud Code Assist wire で使用。ライブ探索は認証済みの CCA `v1internal:fetchAvailableModels` エンドポイントを使用し、ログイン中のアカウントで利用可能な agent モデルのみを公開します。管理されたカタログはフォールバックとして残ります。 | | `cursor` | `cursor` | `https://api2.cursor.sh` | 実験的 PKCE ログイン、HTTP/2 トランスポート、アカウント別モデル探索をサポート。 | | `github-copilot` | `openai-chat` | `https://api.githubcopilot.com` | 実験的。GitHub デバイスフロー + `copilot_internal` 交換（VS Code OAuth クライアント）。有効な Copilot サブスクリプションが必要で、公式のサードパーティ API ではありません。 | diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 1c74f02cf..de93be7cf 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -107,7 +107,7 @@ ocx logout | `anthropic` | `anthropic` | `https://api.anthropic.com` | Claude 모델; 실시간 모델 목록은 `/v1/models`에서 가져옵니다. | | `kimi` | `openai-chat` | `https://api.kimi.com/coding/v1` | Kimi K2.7/K2.6/K2.5 코딩 모델. | | `kiro` | `kiro` | `https://runtime.us-east-1.kiro.dev` | 최초 로그인은 설치하고 로그인한 `kiro-cli` 세션을 가져옵니다(Unix에서는 `curl -fsSL https://cli.kiro.dev/install | bash`, Windows PowerShell에서는 `irm 'https://cli.kiro.dev/install.ps1' | iex`로 설치한 뒤 `kiro-cli login` 실행). **계정 추가**는 `kiro-cli`에서 로그아웃한 뒤 새 브라우저 로그인을 시작하여 `kiro-cli` 자체의 계정을 전환하고, 계정별 프로필 메타데이터를 저장합니다. 기존 OpenCodex 계정은 유지되며, 취소되거나 실패하면 이전 `kiro-cli` 세션을 복원합니다. | -| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth를 Cloud Code Assist wire로 사용합니다. CCA가 범용 `/models` 엔드포인트를 제공하지 않으므로 유지 관리되는 6개 모델 정적 카탈로그를 사용합니다. | +| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth를 Cloud Code Assist wire로 사용합니다. 실시간 탐색은 인증된 CCA `v1internal:fetchAvailableModels` 엔드포인트를 사용하며 로그인한 계정에서 사용할 수 있는 agent 모델만 게시합니다. 유지 관리되는 카탈로그는 폴백으로 남습니다. | | `cursor` | `cursor` | `https://api2.cursor.sh` | 실험적 PKCE 로그인, HTTP/2 전송, 계정별 모델 탐색을 지원합니다. | | `github-copilot` | `openai-chat` | `https://api.githubcopilot.com` | 실험적. GitHub 디바이스 플로우 + `copilot_internal` 교환(VS Code OAuth 클라이언트). 활성 Copilot 구독 필요; 공식 서드파티 API가 아닙니다. | diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 977bff86b..79125612c 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -117,7 +117,7 @@ ocx logout | `anthropic` | `anthropic` | `https://api.anthropic.com` | Модели Claude; актуальный список моделей загружается из `/v1/models`. | | `kimi` | `openai-chat` | `https://api.kimi.com/coding/v1` | Модели Kimi K2.7/K2.6/K2.5 для кодинга. | | `kiro` | `kiro` | `https://runtime.us-east-1.kiro.dev` | Первый вход импортирует существующую сессию после установки Kiro CLI (в Unix: `curl -fsSL https://cli.kiro.dev/install | bash`; в Windows PowerShell: `irm 'https://cli.kiro.dev/install.ps1' | iex`; затем выполните `kiro-cli login`). **Добавить аккаунт** выполняет выход из `kiro-cli`, запускает новый вход через браузер, переключает аккаунт самого `kiro-cli` и сохраняет метаданные профиля отдельно для каждого аккаунта. Существующие аккаунты OpenCodex сохраняются; при отмене или сбое восстанавливается предыдущая сессия `kiro-cli`. | -| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth поверх протокола Cloud Code Assist. Используется поддерживаемый статический каталог из шести моделей, поскольку CCA не предоставляет общий эндпоинт `/models`. | +| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | Google OAuth поверх протокола Cloud Code Assist. Живое обнаружение использует аутентифицированный CCA-эндпоинт `v1internal:fetchAvailableModels` и публикует только agent-модели, доступные текущему аккаунту; поддерживаемый каталог остаётся резервным вариантом. | | `cursor` | `cursor` | `https://api2.cursor.sh` | Экспериментальный PKCE-вход, живой транспорт HTTP/2 и обнаружение моделей с фильтрацией по аккаунту. | | `github-copilot` | `openai-chat` | `https://api.githubcopilot.com` | Экспериментально. Device flow GitHub + обмен `copilot_internal` (OAuth-клиент VS Code). Требуется активная подписка Copilot; это не официальный сторонний API. | diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index 490fd696a..e016f1946 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -98,7 +98,7 @@ ocx logout | `anthropic` | `anthropic` | `https://api.anthropic.com` | Claude 模型；实时模型列表从 `/v1/models` 获取。 | | `kimi` | `openai-chat` | `https://api.kimi.com/coding/v1` | Kimi K2.7/K2.6/K2.5 编程模型。 | | `kiro` | `kiro` | `https://runtime.us-east-1.kiro.dev` | 首次登录会导入已安装并已登录的 Kiro CLI 会话（Unix 使用 `curl -fsSL https://cli.kiro.dev/install | bash`；Windows PowerShell 使用 `irm 'https://cli.kiro.dev/install.ps1' | iex`；然后运行 `kiro-cli login`）。**添加账户**会先退出 `kiro-cli`，再启动新的浏览器登录，从而切换 `kiro-cli` 自身使用的账户，并保存账户范围的配置文件元数据。现有 OpenCodex 账户会保留；如果取消或失败，则恢复之前的 `kiro-cli` 会话。 | -| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | 通过 Cloud Code Assist 协议使用 Google OAuth。由于 CCA 不提供通用 `/models` 端点，因此使用维护中的六模型静态目录。 | +| `google-antigravity` | `google` | `https://daily-cloudcode-pa.googleapis.com` | 通过 Cloud Code Assist 协议使用 Google OAuth。实时发现调用已认证的 CCA `v1internal:fetchAvailableModels` 端点，并仅发布当前登录账户可用的 agent 模型；维护中的目录仍作为回退。 | | `cursor` | `cursor` | `https://api2.cursor.sh` | 实验性 PKCE 登录、HTTP/2 传输和按账号筛选的模型发现。 | | `github-copilot` | `openai-chat` | `https://api.githubcopilot.com` | 实验性。GitHub 设备流 + `copilot_internal` 交换（VS Code OAuth 客户端）。需要有效的 Copilot 订阅；不是官方第三方 API。 | diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index a92b8a539..648b03406 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -7,10 +7,12 @@ import { CODEX_CONFIG_PATH, CODEX_MODELS_CACHE_PATH, DEFAULT_CATALOG_PATH, readR import { clearModelCache, clearProviderDiscoveryStatus, + captureModelCacheGeneration, DEFAULT_MODEL_CACHE_TTL_MS, getFreshCached, getStaleCached, isModelsFetchCoolingDown, + isModelCacheGenerationCurrent, markModelsFetchFailure, markProviderDiscoveryFailed, markProviderDiscoveryOk, @@ -20,6 +22,7 @@ import { } from "../model-cache"; import { buildModelsRequest, + getValidAccessTokenSnapshot, observeActiveOAuthAccessToken, resolveModelsAuthToken, type OAuthActiveTokenObservation, @@ -29,7 +32,8 @@ import { modelInList } from "../../types"; import { CODEX_REASONING_LEVELS, codexEffortRank, configuredReasoningEfforts, modelRecordValue, sanitizeCodexReasoningEfforts } from "../../reasoning-effort"; import { getModelMetadata, getModelMetadataCaseInsensitive, listModelMetadata, resolveMetadataProvider } from "../../generated/model-metadata"; import { enrichProviderFromRegistry, shouldCaseFoldMetadataModelId } from "../../providers/derive"; -import { getProviderRegistryEntry, providerMatchesRegistryTransport } from "../../providers/registry"; +import { effectiveGoogleMode, getProviderRegistryEntry, providerMatchesRegistryTransport } from "../../providers/registry"; +import { parseAntigravityAvailableModels } from "../../providers/antigravity-models"; import { applyProviderContextCap, providerContextCap } from "../../providers/context-cap"; import { routedSlug, slugEquals, slugsEquivalent } from "../../providers/slug-codec"; import { CODEX_GPT5_IDENTITY_LINE } from "../../adapters/identity"; @@ -47,6 +51,7 @@ import type { NormalizedComboConfig } from "../../combos/types"; import { ProviderOutboundPolicyError, providerOutboundGet, + providerOutboundPost, providerRedirectError, } from "../../lib/provider-outbound"; import { redactSecretString } from "../../lib/redact"; @@ -118,6 +123,7 @@ interface ModelsAuthResolution { readonly apiKey: string | undefined; readonly observed: boolean; readonly oauthApiBaseUrl?: string; + readonly oauthProjectId?: string; } type ModelsAuthResolver = @@ -132,7 +138,7 @@ type ModelsAuthResolverFactory = ( ) => ModelsAuthResolver; interface CapturedModelsRequest { - readonly method: "GET"; + readonly method: "GET" | "POST"; readonly url: string; readonly headersWithoutCredential: Readonly>; readonly headersWithCredential: Readonly>; @@ -359,11 +365,12 @@ function captureModelsRequest( : undefined; const withoutCredential = buildModelsRequest(provider, undefined, name, observed); const withCredential = buildModelsRequest(provider, REQUEST_CREDENTIAL_SENTINEL, name, observed); - if (withoutCredential.url !== withCredential.url) { + const method = withoutCredential.method ?? "GET"; + if (withoutCredential.url !== withCredential.url || method !== (withCredential.method ?? "GET")) { throw new TypeError(`Provider model discovery URL for ${name} depends on credential bytes.`); } return detachedFrozen({ - method: "GET" as const, + method, url: withoutCredential.url, headersWithoutCredential: withoutCredential.headers, headersWithCredential: withCredential.headers, @@ -928,6 +935,7 @@ function observedModelsAuthResolver( apiKey: observation.snapshot.accessToken, observed: true, ...(observation.snapshot.apiBaseUrl ? { oauthApiBaseUrl: observation.snapshot.apiBaseUrl } : {}), + ...(observation.snapshot.projectId ? { oauthProjectId: observation.snapshot.projectId } : {}), }; }, }; @@ -944,6 +952,10 @@ async function fetchProviderModelsWithAuth( models: CatalogModel[], state: CatalogGatherProviderModelOutcome["state"], ): ProviderModelsResult => ({ models, outcome: { provider: name, state } }); + // Capture before any credential refresh or outbound await. OAuth account changes clear this + // generation, so a request started with the former account cannot later publish its result. + const cacheGeneration = captureModelCacheGeneration(name); + const isCurrentCacheGeneration = () => isModelCacheGenerationCurrent(name, cacheGeneration); if (prov.authMode === "forward") return observed([], "authoritative"); // ChatGPT backend has no /models const seedVertexDefault = prov.adapter === "google" && prov.googleMode === "vertex" @@ -962,7 +974,15 @@ async function fetchProviderModelsWithAuth( return observed(configured, "authoritative"); } const auth: ModelsAuthResolution = captured.observedAuth ?? (resolveAuth.kind === "refreshing" - ? { apiKey: await resolveModelsAuthToken(name, prov), observed: false } + ? prov.authMode === "oauth" && effectiveGoogleMode(name, prov) === "cloud-code-assist" + ? await getValidAccessTokenSnapshot(name) + .then(snapshot => ({ + apiKey: snapshot.accessToken, + observed: false, + ...(snapshot.projectId ? { oauthProjectId: snapshot.projectId } : {}), + })) + .catch(() => ({ apiKey: undefined, observed: false })) + : { apiKey: await resolveModelsAuthToken(name, prov), observed: false } : resolveAuth.resolve(name, prov)); const apiKey = auth.apiKey; // A configured default is a real callable selector and must remain discoverable when a @@ -1004,15 +1024,17 @@ async function fetchProviderModelsWithAuth( const available = filterCursorConfiguredModelsByLiveDiscovery(configured, liveResult.models); const result = available.length > 0 ? available : configured; // Count what discovery actually returned, not the configured rows we fall back to. + if (!setCached(name, result, Date.now(), cacheGeneration)) return observed(configured, "degraded"); markProviderDiscoveryOk(name, liveResult.models.length); - setCached(name, result); return observed(result, "authoritative"); } - markModelsFetchFailure(name); - markProviderDiscoveryFailed(name, { reason: "provider" }); - console.warn( - `[opencodex] Cursor model discovery for "${name}" failed [${liveResult.error}]${liveResult.detail ? `: ${liveResult.detail}` : ""}; using stale/static catalog degradation.`, - ); + if (isCurrentCacheGeneration()) { + markModelsFetchFailure(name); + markProviderDiscoveryFailed(name, { reason: "provider" }); + console.warn( + `[opencodex] Cursor model discovery for "${name}" failed [${liveResult.error}]${liveResult.detail ? `: ${liveResult.detail}` : ""}; using stale/static catalog degradation.`, + ); + } const staleCursor = getStaleCached(name); return observed( staleCursor ? applyConfigHintsToCachedModels(name, prov, staleCursor) : configured, @@ -1025,6 +1047,9 @@ async function fetchProviderModelsWithAuth( // matching Cursor's !apiKey → configured degradation and fetch-failure fallback. return observed(configured, "degraded"); } + const cloudCodeAssist = effectiveGoogleMode(name, prov) === "cloud-code-assist"; + const project = prov.project ?? auth.oauthProjectId; + if (cloudCodeAssist && !project) return observed(configured, "degraded"); const fresh = getFreshCached(name, ttlMs); if (fresh) { return observed( @@ -1051,6 +1076,9 @@ async function fetchProviderModelsWithAuth( const failedDiscoveryFallback = ( failure: ProviderModelDiscoveryFailure, ): { models: CatalogModel[]; fallback: "stale" | "configured"; shouldLog: boolean } => { + if (!isCurrentCacheGeneration()) { + return { models: failedDiscoveryConfigured, fallback: "configured", shouldLog: false }; + } // Decide logging BEFORE recording the new status, so we can compare against the prior one and // suppress an identical repeated failure (#395 log flood). The failure stays observable via the // discovery-status API regardless. @@ -1067,10 +1095,16 @@ async function fetchProviderModelsWithAuth( }; }; try { - const res = await providerOutboundGet(name, prov, url, { - headers, - signal: AbortSignal.timeout(8000), - }); + const res = request.method === "POST" + ? await providerOutboundPost(name, prov, url, { + headers, + body: JSON.stringify({ project }), + signal: AbortSignal.timeout(8000), + }) + : await providerOutboundGet(name, prov, url, { + headers, + signal: AbortSignal.timeout(8000), + }); const redirectError = await providerRedirectError(res, url); if (redirectError) { const { models, fallback, shouldLog } = failedDiscoveryFallback({ reason: "http", httpStatus: res.status }); @@ -1109,6 +1143,32 @@ async function fetchProviderModelsWithAuth( } return observed(models, "degraded"); } + const antigravity = cloudCodeAssist + ? parseAntigravityAvailableModels(bounded.value, discovery.maxModels) + : undefined; + if (cloudCodeAssist && !antigravity) { + const { models, fallback, shouldLog } = failedDiscoveryFallback({ reason: "invalid_response" }); + if (shouldLog) { + console.warn( + `[opencodex] Provider model discovery for "${name}" returned malformed CCA model data [status=${res.status}, contentType=${contentType}, urlClass=${urlClass}, fallback=${fallback}].`, + ); + } + return observed(models, "degraded"); + } + if (antigravity) { + const live = antigravity.map(model => applyProviderConfigHints(name, prov, { + id: model.id, + provider: name, + // CCA only exposes a numeric thinking budget. Until the adapter owns an exact Codex + // effort-to-wire mapping for a newly discovered model, do not advertise a false ladder. + reasoningEfforts: [], + ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}), + ...(model.inputModalities ? { inputModalities: model.inputModalities } : {}), + }, contextCap)); + if (!setCached(name, live, Date.now(), cacheGeneration)) return observed(configured, "degraded"); + markProviderDiscoveryOk(name, live.length); + return observed(live, "authoritative"); + } const extracted = extractProviderModelItems(bounded.value, discovery); if (!extracted.ok) { const { models, fallback, shouldLog } = failedDiscoveryFallback({ reason: "invalid_response" }); @@ -1167,8 +1227,8 @@ async function fetchProviderModelsWithAuth( && !QUIET_AUTHORITATIVE_CATALOG_PROVIDERS.has(name)) { warnDroppedConfiguredIdsOnce(name, droppedConfiguredIds); } + if (!setCached(name, live, Date.now(), cacheGeneration)) return observed(configured, "degraded"); markProviderDiscoveryOk(name, liveModelCount); - setCached(name, live); return observed(live, "authoritative"); } catch (error) { if (error instanceof ProviderOutboundPolicyError) { diff --git a/src/codex/catalog/sync.ts b/src/codex/catalog/sync.ts index 1eeb890ff..d9fae24c0 100644 --- a/src/codex/catalog/sync.ts +++ b/src/codex/catalog/sync.ts @@ -466,7 +466,7 @@ export function resetCatalogRuntimeStateForTests(): void { comboUnrestorableShadowWarnings.clear(); accountSelectorShadowCollisionWarnings.clear(); clearLastComboCatalogOmissions(); - clearModelCache(); + clearModelCache(undefined, "eviction"); clearGatherRoutedModelsInflight(); } diff --git a/src/codex/convergence-types.ts b/src/codex/convergence-types.ts index 06bd62e48..515ee8c94 100644 --- a/src/codex/convergence-types.ts +++ b/src/codex/convergence-types.ts @@ -422,7 +422,7 @@ export interface CatalogProviderDiscoveryPolicySnapshot { readonly path: CatalogDiscoveryPolicyField; readonly query: CatalogDiscoveryPolicyField> | undefined>; }>; - readonly finalMethod: "GET"; + readonly finalMethod: "GET" | "POST"; readonly finalUrl: string; readonly filter: CatalogDiscoveryPolicyField; readonly maxResponseBytes: number; diff --git a/src/codex/model-cache.ts b/src/codex/model-cache.ts index e6d391679..af6eb67e6 100644 --- a/src/codex/model-cache.ts +++ b/src/codex/model-cache.ts @@ -42,7 +42,12 @@ export type ProviderModelDiscoveryFailure = ProviderModelDiscoveryStatus extends : never : never; +/** Whether clearing cache rows also revokes in-flight discovery authority. */ +export type ModelCacheClearReason = "authority" | "eviction"; + const cache = new Map(); +let globalCacheGeneration = 0; +const providerCacheGenerations = new Map(); let cacheBytes = 0; let oldestCachedProvider: string | undefined; let oldestCachedAt: number | null = null; @@ -155,7 +160,29 @@ export function getStaleCached(provider: string): CatalogModel[] | null { return cache.get(provider)?.models ?? null; } -export function setCached(provider: string, models: CatalogModel[], now = Date.now()): void { +/** Capture the cache generation before an asynchronous provider discovery starts. */ +export function captureModelCacheGeneration(provider: string): string { + if (!providerCacheGenerations.has(provider)) providerCacheGenerations.set(provider, 0); + return `${globalCacheGeneration}:${providerCacheGenerations.get(provider)!}`; +} + +/** Whether a discovery started under {@link captureModelCacheGeneration} may still write. */ +export function isModelCacheGenerationCurrent(provider: string, generation: string): boolean { + return generation === captureModelCacheGeneration(provider); +} + +/** + * Store a live result unless the cache was cleared while that asynchronous discovery was running. + * The optional generation keeps existing direct cache writers unchanged while discovery callers can + * prevent a previous OAuth account from repopulating the current account's cache. + */ +export function setCached( + provider: string, + models: CatalogModel[], + now = Date.now(), + generation?: string, +): boolean { + if (generation !== undefined && !isModelCacheGenerationCurrent(provider, generation)) return false; deleteCachedProvider(provider); const sizeBytes = modelCacheEncoder.encode(provider).byteLength + modelCacheEncoder.encode(JSON.stringify(models)).byteLength; @@ -166,16 +193,25 @@ export function setCached(provider: string, models: CatalogModel[], now = Date.n oldestCachedAt = now; } enforceAppOwnedMemoryBudget(); + return true; } /** Drop one provider's cache (or all) so the next resolve forces a live re-fetch. */ -export function clearModelCache(provider?: string): void { +export function clearModelCache( + provider?: string, + reason: ModelCacheClearReason = "authority", +): void { + const revokesInFlightDiscovery = reason === "authority"; if (provider) { + if (revokesInFlightDiscovery) { + providerCacheGenerations.set(provider, (providerCacheGenerations.get(provider) ?? 0) + 1); + } deleteCachedProvider(provider); failureAt.delete(provider); discoveryStatus.delete(provider); liveModelCounts.delete(provider); } else { + if (revokesInFlightDiscovery) globalCacheGeneration += 1; cache.clear(); cacheBytes = 0; oldestCachedProvider = undefined; @@ -192,16 +228,20 @@ export function reconcileModelCacheProviders( ): number { if (generation <= lastReconciledGeneration) return 0; const removedProviders = new Set(); - for (const store of [failureAt, discoveryStatus, liveModelCounts]) { - for (const provider of store.keys()) { - if (validProviders.has(provider)) continue; - store.delete(provider); - removedProviders.add(provider); - } - } - for (const provider of cache.keys()) { + const trackedProviders = new Set([ + ...providerCacheGenerations.keys(), + ...failureAt.keys(), + ...discoveryStatus.keys(), + ...liveModelCounts.keys(), + ...cache.keys(), + ]); + for (const provider of trackedProviders) { if (validProviders.has(provider)) continue; + providerCacheGenerations.set(provider, (providerCacheGenerations.get(provider) ?? 0) + 1); deleteCachedProvider(provider); + failureAt.delete(provider); + discoveryStatus.delete(provider); + liveModelCounts.delete(provider); removedProviders.add(provider); } lastReconciledGeneration = generation; diff --git a/src/config.ts b/src/config.ts index 15e99f28b..b662959bd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1057,9 +1057,8 @@ const configSchema = z.object({ providers: z.record(z.string(), providerConfigSchema), defaultProvider: z.string().min(1).default("openai"), openaiProviderTierVersion: z.union([z.literal(1), z.literal(2)]).optional(), - // Invalid hand edits must not discard an otherwise usable config. Treat them as - // pre-migration so startup can safely re-run the one-time normalization. - googleAntigravityStaticCatalogVersion: z.literal(1).optional().catch(undefined), + // Invalid hand edits must not discard an otherwise usable config. + googleAntigravityStaticCatalogVersion: z.union([z.literal(1), z.literal(2)]).optional().catch(undefined), clientIntegrations: clientIntegrationsSchema.optional().catch(undefined), providerContextCaps: z.record(z.string(), z.number().int().positive()).optional(), contextCapValue: z.number().int().positive().optional(), @@ -2004,8 +2003,8 @@ function googleAntigravityStaticCatalogVersionError(value: unknown): string | nu const raw = rawConfigRecord(value); if (!raw || !Object.hasOwn(raw, "googleAntigravityStaticCatalogVersion")) return null; const version = raw.googleAntigravityStaticCatalogVersion; - if (version === undefined || version === 1) return null; - return "schema_invalid: googleAntigravityStaticCatalogVersion: must be 1 or omitted"; + if (version === undefined || version === 1 || version === 2) return null; + return "schema_invalid: googleAntigravityStaticCatalogVersion: must be 1, 2, or omitted"; } function codexAccountPickerEnabledError(value: unknown): string | null { diff --git a/src/lib/pinned-http.ts b/src/lib/pinned-http.ts index 0a6ecf291..0b7123e4a 100644 --- a/src/lib/pinned-http.ts +++ b/src/lib/pinned-http.ts @@ -3,7 +3,7 @@ import https from "node:https"; export type PinnedAddress = { address: string; family: number }; -export interface PinnedHttpGetOptions { +export interface PinnedHttpRequestOptions { headers?: HeadersInit; maxBytes?: number; idleTimeoutMs?: number; @@ -11,15 +11,16 @@ export interface PinnedHttpGetOptions { context?: string; } -/** - * GET a URL through one previously validated address. The original hostname - * remains authoritative for Host, SNI, and certificate verification. - */ -export function pinnedHttpGet( +/** @deprecated Use {@link PinnedHttpRequestOptions}. */ +export type PinnedHttpGetOptions = PinnedHttpRequestOptions; + +function pinnedHttpRequest( url: string, pinned: PinnedAddress, + method: "GET" | "POST", + body: string | undefined, signal?: AbortSignal, - options?: PinnedHttpGetOptions, + options?: PinnedHttpRequestOptions, ): Promise { const parsed = new URL(url); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { @@ -30,6 +31,9 @@ export function pinnedHttpGet( const maxBytes = options?.maxBytes; const headers = new Headers(options?.headers); headers.set("host", parsed.host); + if (body !== undefined && !headers.has("content-length")) { + headers.set("content-length", String(Buffer.byteLength(body))); + } const requestHeaders: Record = {}; headers.forEach((value, key) => { requestHeaders[key] = value; }); @@ -51,7 +55,7 @@ export function pinnedHttpGet( hostname: parsed.hostname, port: parsed.port || (parsed.protocol === "https:" ? 443 : 80), path: `${parsed.pathname}${parsed.search}`, - method: "GET", + method, headers: requestHeaders, ...(parsed.protocol === "https:" ? { @@ -146,6 +150,33 @@ export function pinnedHttpGet( fail(error); }); req.on("close", () => signal?.removeEventListener("abort", onAbort)); - req.end(); + req.end(body); }); } + +/** + * GET a URL through one previously validated address. The original hostname + * remains authoritative for Host, SNI, and certificate verification. + */ +export function pinnedHttpGet( + url: string, + pinned: PinnedAddress, + signal?: AbortSignal, + options?: PinnedHttpRequestOptions, +): Promise { + return pinnedHttpRequest(url, pinned, "GET", undefined, signal, options); +} + +/** + * POST a string body through one previously validated address. The original + * hostname remains authoritative for Host, SNI, and certificate verification. + */ +export function pinnedHttpPost( + url: string, + pinned: PinnedAddress, + body: string, + signal?: AbortSignal, + options?: PinnedHttpRequestOptions, +): Promise { + return pinnedHttpRequest(url, pinned, "POST", body, signal, options); +} diff --git a/src/lib/provider-outbound.ts b/src/lib/provider-outbound.ts index 29e712623..67b55b470 100644 --- a/src/lib/provider-outbound.ts +++ b/src/lib/provider-outbound.ts @@ -6,17 +6,19 @@ import { providerDestinationConfigError, resolvePublicAddresses, } from "./destination-policy"; -import { pinnedHttpGet } from "./pinned-http"; +import { pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; import { outboundProxyConfigured } from "./proxy-env"; import { publicProviderBaseUrl } from "./provider-url"; type ProviderGetInit = Omit; +type ProviderPostInit = ProviderGetInit & { body: string }; type ProviderOutboundConfig = Pick & { fetch?: typeof globalThis.fetch; }; export interface ProviderOutboundDependencies { resolveAddresses?: typeof resolvePublicAddresses; pinnedGet?: typeof pinnedHttpGet; + pinnedPost?: typeof pinnedHttpPost; } export class ProviderOutboundPolicyError extends Error { @@ -102,13 +104,18 @@ export async function providerRedirectError(response: Response, requestUrl: stri return `provider returned ${response.status} redirect to ${target}; configure the final provider URL directly`; } -export async function providerOutboundGet( +async function providerOutboundRequest( name: string, provider: ProviderOutboundConfig, url: string, - init: ProviderGetInit = {}, + method: "GET" | "POST", + init: ProviderGetInit | ProviderPostInit, dependencies: ProviderOutboundDependencies = {}, ): Promise { + const postUrl = method === "POST" ? new URL(url) : undefined; + if (postUrl?.protocol !== undefined && postUrl.protocol !== "https:") { + throw new ProviderOutboundPolicyError("provider POST URL must use HTTPS"); + } if (provider.fetch) { // A caller-owned executor cannot be peer-pinned here. This branch keeps literal/config // checks and redirect blocking, but does not provide the resolved-address guarantees of @@ -128,12 +135,13 @@ export async function providerOutboundGet( }); if (destinationError) throw new ProviderOutboundPolicyError(destinationError); } - return provider.fetch(url, { ...init, method: "GET", redirect: "manual" }); + return provider.fetch(url, { ...init, method, redirect: "manual" }); } - const parsed = new URL(url); + const parsed = postUrl ?? new URL(url); const proxyConfigured = configuredProxyFor(); const resolveAddresses = dependencies.resolveAddresses ?? resolvePublicAddresses; const pinnedGet = dependencies.pinnedGet ?? pinnedHttpGet; + const pinnedPost = dependencies.pinnedPost ?? pinnedHttpPost; const allowPrivate = providerAllowsPrivateNetwork(name, provider); let resolved: Awaited>; try { @@ -150,11 +158,11 @@ export async function providerOutboundGet( if (!proxyConfigured) throw error; warnProxyBoundaryOnce(); warnProxyDnsDegradationOnce(); - return globalThis.fetch(url, { ...init, method: "GET", redirect: "manual" }); + return globalThis.fetch(url, { ...init, method, redirect: "manual" }); } if (proxyConfigured && !resolved.privateNetwork) { warnProxyBoundaryOnce(); - return globalThis.fetch(url, { ...init, method: "GET", redirect: "manual" }); + return globalThis.fetch(url, { ...init, method, redirect: "manual" }); } if (proxyConfigured && resolved.privateNetwork && !noProxyMatches(parsed)) { const hostname = normalizeProxyHostname(parsed.hostname); @@ -162,9 +170,34 @@ export async function providerOutboundGet( `provider URL resolves to a private-network destination; add ${hostname} to NO_PROXY before using allowPrivateNetwork with an outbound proxy`, ); } - return pinnedGet(url, pickPinnedAddress(resolved.addresses), init.signal ?? undefined, { + const requestOptions = { headers: init.headers, rejectUnauthorized: true, context: "provider response", - }); + }; + const pinned = pickPinnedAddress(resolved.addresses); + if (method === "POST") { + return pinnedPost(url, pinned, (init as ProviderPostInit).body, init.signal ?? undefined, requestOptions); + } + return pinnedGet(url, pinned, init.signal ?? undefined, requestOptions); +} + +export async function providerOutboundGet( + name: string, + provider: ProviderOutboundConfig, + url: string, + init: ProviderGetInit = {}, + dependencies: ProviderOutboundDependencies = {}, +): Promise { + return providerOutboundRequest(name, provider, url, "GET", init, dependencies); +} + +export async function providerOutboundPost( + name: string, + provider: ProviderOutboundConfig, + url: string, + init: ProviderPostInit, + dependencies: ProviderOutboundDependencies = {}, +): Promise { + return providerOutboundRequest(name, provider, url, "POST", init, dependencies); } diff --git a/src/oauth/index.ts b/src/oauth/index.ts index a8b291141..778ba3306 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -13,6 +13,7 @@ import { loginAntigravity, refreshAntigravityToken } from "./google-antigravity" import { loginCursor, refreshCursorToken } from "./cursor"; import { loginGithubCopilot, refreshGithubCopilotToken, validateCopilotApiBaseUrl } from "./github-copilot"; import { loginCommandCode, refreshCommandCodeToken } from "./command-code"; +import { ANTIGRAVITY_REQUEST_UA } from "../adapters/google-antigravity-wire"; import { deriveOAuthDefaultModel, deriveOAuthProviderConfig } from "../providers/derive"; import { apiKeyPoolEntryId, sanitizeApiKeyValue } from "../providers/api-keys"; import { effectiveGoogleMode, getProviderRegistryEntry, providerMatchesRegistryTransport } from "../providers/registry"; @@ -53,6 +54,8 @@ export interface OAuthAccessSnapshot { accountId: string; generation: string; accessToken: string; + /** Cloud Code Assist project selected during Antigravity login. */ + projectId?: string; /** Safe request-routing subset; refresh-only Kiro client secrets never leave the credential store. */ kiro?: Pick; } @@ -289,6 +292,7 @@ function accessSnapshot(provider: string, accountId: string, cred: OAuthCredenti accountId, generation: credentialGeneration(cred), accessToken: cred.access, + ...(cred.projectId ? { projectId: cred.projectId } : {}), // Stored account metadata remains authoritative. Metadata-less legacy/environment credentials // may use explicit environment routing, but never borrow the currently signed-in local CLI account. ...(provider === "kiro" @@ -628,15 +632,15 @@ function modelDiscoveryTransportSeed(providerName: string, prov: OcxProviderConf } /** - * Provider-correct `GET /models` request (URL + headers), so both model-listing paths fetch the + * Provider-correct model-discovery request (URL + headers), so both model-listing paths fetch the * LIVE catalog correctly per adapter. Anthropic is the special case: its endpoint is `/v1/models` * (not `/models`), it needs `anthropic-version`, and it authenticates with `x-api-key` by default * (or `Authorization: Bearer` when `apiKeyTransport = "bearer"`), plus the OAuth beta for oauth * mode — not a bare Bearer. Google (ai-studio mode) * is the other special case: `x-goog-api-key` + `/v1beta/models`, returning `{ models: [...] }`. * The catalog authority gate intentionally degrades that non-OpenAI shape to stale/static data. - * Everyone else uses the OpenAI-style `/models` + Bearer with a `{ data: [{ id, owned_by? }] }` - * response. + * Antigravity uses its CCA `:fetchAvailableModels` RPC; everyone else uses the OpenAI-style + * `/models` + Bearer with a `{ data: [{ id, owned_by? }] }` response. */ export interface ModelsRequestObservedAuth { readonly oauthApiBaseUrl?: string; @@ -647,7 +651,7 @@ export function buildModelsRequest( apiKey: string | undefined, providerName = "", observedAuth?: ModelsRequestObservedAuth, -): { url: string; headers: Record } { +): { method?: "POST"; url: string; headers: Record } { const transportSeed = modelDiscoveryTransportSeed(providerName, prov); const copilotApiBaseUrl = observedAuth === undefined ? (providerName === "github-copilot" ? getOAuthCredentialApiBaseUrl(providerName) : undefined) @@ -665,6 +669,17 @@ export function buildModelsRequest( effectiveProvider.baseUrl, defaultUrl, ); + if (effectiveGoogleMode(providerName, effectiveProvider) === "cloud-code-assist") { + headers.Accept = "application/json"; + headers["Content-Type"] = "application/json"; + headers["User-Agent"] = ANTIGRAVITY_REQUEST_UA; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + return { + method: "POST", + url: discoveryUrl(`${effectiveProvider.baseUrl.replace(/\/+$/, "")}/v1internal:fetchAvailableModels`), + headers, + }; + } if (effectiveGoogleMode(providerName, effectiveProvider) === "ai-studio") { // Generative Language API: API key goes in x-goog-api-key (never Authorization: Bearer), // models live under /v1beta (v1 misses preview models), and pageSize maxes at 1000 — @@ -698,9 +713,7 @@ export function buildModelsRequest( * * Only touches providers that are registry-managed AND still `authMode: "oauth"`. Preset fields * are refreshed, while the registry's `liveModels` default is normally filled only when no value - * is stored. Antigravity has one versioned exception below because its old GUI-generated `true` - * cannot be distinguished from a hand-written pre-migration `true`. Persists + returns true when - * anything changed. + * is stored. Persists + returns true when anything changed. */ function cloneProviderField(value: unknown): unknown { if (Array.isArray(value)) return [...value]; @@ -729,7 +742,7 @@ const OAUTH_RECONCILE_FIELDS: (keyof OcxProviderConfig)[] = [ ]; const GOOGLE_ANTIGRAVITY_PROVIDER = "google-antigravity"; -const GOOGLE_ANTIGRAVITY_STATIC_CATALOG_VERSION = 1 as const; +const GOOGLE_ANTIGRAVITY_LIVE_DISCOVERY_VERSION = 2 as const; /** Only migrate the three-model experimental seed; an operator's later `liveModels: false` wins. */ function isLegacyCommandCodeStaticCatalog(provider: OcxProviderConfig): boolean { @@ -738,10 +751,34 @@ function isLegacyCommandCodeStaticCatalog(provider: OcxProviderConfig): boolean && JSON.stringify(provider.models) === JSON.stringify(["deepseek-v4-flash", "kimi-k3", "glm-5.2"]); } +function isLegacyAntigravityStaticCatalog(provider: OcxProviderConfig): boolean { + return provider.liveModels === false + && provider.adapter === "google" + && provider.baseUrl === "https://daily-cloudcode-pa.googleapis.com" + && provider.authMode === "oauth" + && provider.googleMode === "cloud-code-assist" + && provider.defaultModel === "gemini-3.6-flash" + && JSON.stringify(provider.models) === JSON.stringify([ + "gemini-3.6-flash", + "gemini-3.1-pro", + "gemini-3.1-flash-image", + "claude-sonnet-4-6", + "claude-opus-4-6-thinking", + "gpt-oss-120b-medium", + ]); +} + +/** Promote only the versioned canonical static seed; unmarked `liveModels: false` remains user intent. */ +function migrateLegacyAntigravityStaticCatalog(config: OcxConfig): boolean { + if (config.googleAntigravityStaticCatalogVersion !== 1) return false; + const provider = config.providers[GOOGLE_ANTIGRAVITY_PROVIDER]; + if (provider && isLegacyAntigravityStaticCatalog(provider)) provider.liveModels = true; + config.googleAntigravityStaticCatalogVersion = GOOGLE_ANTIGRAVITY_LIVE_DISCOVERY_VERSION; + return true; +} + export function reconcileOAuthProviders(config: OcxConfig): boolean { - let changed = false; - const migrateAntigravityStaticCatalog = - config.googleAntigravityStaticCatalogVersion !== GOOGLE_ANTIGRAVITY_STATIC_CATALOG_VERSION; + let changed = migrateLegacyAntigravityStaticCatalog(config); for (const [name, prov] of Object.entries(config.providers)) { const def = OAUTH_PROVIDERS[name]; if (name === "command-code" && isLegacyCommandCodeStaticCatalog(prov)) { @@ -750,21 +787,7 @@ export function reconcileOAuthProviders(config: OcxConfig): boolean { prov.liveModels = true; changed = true; } - // Normalize the canonical row before the OAuth-only reconciliation guard. The old GUI and a - // manual edit both persist the same bare `true`, with no source metadata, so every ambiguous - // pre-marker value is reset once. A deliberate live-discovery choice can be re-enabled after - // the marker and is then preserved. Do this before the guard so omitted/non-OAuth authMode - // rows do not get stamped without actually receiving the new static default. - if (name === GOOGLE_ANTIGRAVITY_PROVIDER && migrateAntigravityStaticCatalog && prov.liveModels !== false) { - prov.liveModels = false; - changed = true; - } - // During the one-time Antigravity static-catalog migration, also refresh preset catalog - // fields when authMode is omitted or non-oauth. Otherwise liveModels flips to static while - // a stale models[] remains the published catalog forever. - const migrateAntigravityCatalogFields = - name === GOOGLE_ANTIGRAVITY_PROVIDER && migrateAntigravityStaticCatalog; - if (!def || (prov.authMode !== "oauth" && !migrateAntigravityCatalogFields)) continue; + if (!def || prov.authMode !== "oauth") continue; const preset = def.providerConfig; for (const field of OAUTH_RECONCILE_FIELDS) { if (JSON.stringify(prov[field]) === JSON.stringify(preset[field])) continue; @@ -775,9 +798,6 @@ export function reconcileOAuthProviders(config: OcxConfig): boolean { } changed = true; } - // Before this marker existed, the GUI materialized an omitted `liveModels` as `true` on any - // settings save. Since persisted values have no provenance, the pre-guard normalization above - // intentionally resets all pre-marker `true` values once. Later choices are version-bounded. if (prov.liveModels === undefined && preset.liveModels !== undefined) { prov.liveModels = preset.liveModels; changed = true; @@ -791,10 +811,6 @@ export function reconcileOAuthProviders(config: OcxConfig): boolean { changed = true; } } - if (migrateAntigravityStaticCatalog) { - config.googleAntigravityStaticCatalogVersion = GOOGLE_ANTIGRAVITY_STATIC_CATALOG_VERSION; - changed = true; - } if (changed) saveConfig(config); return changed; } @@ -851,17 +867,14 @@ export function upsertOAuthProvider(config: OcxConfig, provider: string): void { if (provider === "chatgpt") return; const def = OAUTH_PROVIDERS[provider]; if (!def) return; + if (provider === GOOGLE_ANTIGRAVITY_PROVIDER) migrateLegacyAntigravityStaticCatalog(config); const namespaceCollision = codexAccountNamespaceProviderCollisionError(config.codexAccountNamespaces, provider); if (namespaceCollision) throw new Error(namespaceCollision); const existing = config.providers[provider]; const next: OcxProviderConfig = { ...def.providerConfig }; - // `liveModels` is a user-facing provider toggle. A registry default seeds new rows, but an - // explicit post-migration choice must survive re-login and the latest-config upsert. Old GUI - // saves and manual edits left identical pre-marker `true` values, so that ambiguous state is - // reset once; users who deliberately forced discovery can re-enable it after migration. - const preserveExistingLiveModels = provider !== GOOGLE_ANTIGRAVITY_PROVIDER - || config.googleAntigravityStaticCatalogVersion === GOOGLE_ANTIGRAVITY_STATIC_CATALOG_VERSION; - if (preserveExistingLiveModels && typeof existing?.liveModels === "boolean" && !isLegacyCommandCodeStaticCatalog(existing)) { + // `liveModels` is a user-facing provider toggle. Preserve either explicit setting across login; + // Antigravity's CCA discovery now uses its real RPC, so legacy `true` remains a valid choice. + if (typeof existing?.liveModels === "boolean" && !isLegacyCommandCodeStaticCatalog(existing)) { next.liveModels = existing.liveModels; } // The Command Code protocol-version pin is an operator compatibility control. A re-login, @@ -893,9 +906,6 @@ export function upsertOAuthProvider(config: OcxConfig, provider: string): void { } } config.providers[provider] = next; - if (provider === GOOGLE_ANTIGRAVITY_PROVIDER) { - config.googleAntigravityStaticCatalogVersion = GOOGLE_ANTIGRAVITY_STATIC_CATALOG_VERSION; - } } interface RunLoginDeps { @@ -1023,11 +1033,15 @@ export async function runLogin( settleKiroTransaction(rawCred, true); if (provider !== "chatgpt") { try { + const { clearModelCache } = await import("../codex/model-cache"); + const { clearGatherRoutedModelsInflight } = await import("../codex/catalog"); + clearModelCache(provider); + clearGatherRoutedModelsInflight(); const { clearAccountQuotaCache, clearProviderQuotaCache } = await import("../providers/quota"); clearProviderQuotaCache(); clearAccountQuotaCache(provider); } catch { - // Quota module may be unavailable in tightly scoped unit tests. + // Optional state modules may be unavailable in tightly scoped unit tests. } } return cred; diff --git a/src/providers/antigravity-models.ts b/src/providers/antigravity-models.ts index 18b0b3961..67dcf84ff 100644 --- a/src/providers/antigravity-models.ts +++ b/src/providers/antigravity-models.ts @@ -1,10 +1,13 @@ +import { isValidModelDiscoveryModelId, MODEL_DISCOVERY_MAX_MODELS } from "./model-discovery-limits"; + // Google Antigravity (Cloud Code Assist) bundled model list. // // Single source of truth: the Antigravity `:fetchAvailableModels` backend, the same one the `agy` // CLI resolves labels against. The ids below separate CCA wire ids, collapsed picker entries, // and hidden compatibility aliases for saved selections. The CCA envelope's `model` field must // receive the wire id (for example "Gemini 3.1 Pro (High)" => gemini-pro-agent), while the -// picker exposes collapsed base models with reasoning-effort routing. +// picker exposes collapsed base models only when CCA returns every known tier; otherwise each +// returned wire id remains visible so an unavailable tier cannot be selected. // ── Wire IDs (what CCA :fetchAvailableModels returns) ── const ANTIGRAVITY_WIRE_MODELS = [ @@ -19,6 +22,21 @@ const ANTIGRAVITY_WIRE_MODELS = [ "gpt-oss-120b-medium", ]; +const ANTIGRAVITY_PICKER_MODEL_BY_WIRE_ID: Record = { + "gemini-3.6-flash-low": "gemini-3.6-flash", + "gemini-3.6-flash-medium": "gemini-3.6-flash", + "gemini-3.6-flash-high": "gemini-3.6-flash", + "gemini-3.1-pro-low": "gemini-3.1-pro", + "gemini-pro-agent": "gemini-3.1-pro", +}; + +const ANTIGRAVITY_WIRE_IDS_BY_PICKER_MODEL: Record = Object.entries( + ANTIGRAVITY_PICKER_MODEL_BY_WIRE_ID, +).reduce>((out, [wireId, pickerId]) => { + (out[pickerId] ??= []).push(wireId); + return out; +}, {}); + // ── Effort ladders per collapsed base model ── // Gemini models: effort → wire model suffix (official agy UI pattern). // Claude Opus: effort → thinkingConfig.thinkingLevel (CLIProxyAPI proven pattern). @@ -100,8 +118,8 @@ const ANTIGRAVITY_WIRE_MODEL_CONTEXT_WINDOWS: Record = { "gemini-3.1-pro-low": 1_048_576, "gemini-pro-agent": 1_048_576, "gemini-3.1-flash-image": 1_048_576, - "claude-sonnet-4-6": 200_000, - "claude-opus-4-6-thinking": 1_000_000, + "claude-sonnet-4-6": 250_000, + "claude-opus-4-6-thinking": 250_000, "gpt-oss-120b-medium": 131_072, }; @@ -119,6 +137,100 @@ export const ANTIGRAVITY_MODEL_CONTEXT_WINDOWS: Record = { ), }; +export const ANTIGRAVITY_MODEL_INPUT_MODALITIES: Record = { + "gemini-3.6-flash": ["text", "image"], + "gemini-3.1-pro": ["text", "image"], + "gemini-3.1-flash-image": ["text", "image"], + "claude-sonnet-4-6": ["text", "image"], + "claude-opus-4-6-thinking": ["text", "image"], + "gpt-oss-120b-medium": ["text"], +}; + +export interface AntigravityAvailableModel { + id: string; + contextWindow?: number; + inputModalities?: string[]; +} + +function antigravityRecord(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +function antigravityPositiveInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; +} + +/** + * Extract the CCA models that are valid for agent requests. The endpoint also returns tab, + * command, commit-message, transcription, and standalone image-generation models; those are not + * callable through the CCA agent envelope and must not be published to the Codex catalog. + */ +export function parseAntigravityAvailableModels( + payload: unknown, + maxModels = MODEL_DISCOVERY_MAX_MODELS, +): AntigravityAvailableModel[] | null { + const limit = Number.isSafeInteger(maxModels) && maxModels > 0 + ? Math.min(maxModels, MODEL_DISCOVERY_MAX_MODELS) + : MODEL_DISCOVERY_MAX_MODELS; + const body = antigravityRecord(payload); + if (!body) return null; + const models = antigravityRecord(body?.models); + const sorts = Array.isArray(body?.agentModelSorts) ? body.agentModelSorts : undefined; + if (!models || !sorts) return null; + + const ids: string[] = []; + for (const sort of sorts) { + const groups = antigravityRecord(sort)?.groups; + if (!Array.isArray(groups)) return null; + for (const group of groups) { + const modelIds = antigravityRecord(group)?.modelIds; + if (!Array.isArray(modelIds)) return null; + for (const id of modelIds) { + if (!isValidModelDiscoveryModelId(id) + || !antigravityRecord(models[id]) + || ids.length >= limit) return null; + ids.push(id); + } + } + } + // This model is exposed by Antigravity's agent chat surface even though it is grouped under + // image generation in the discovery response. + if (Array.isArray(body.imageGenerationModelIds) + && body.imageGenerationModelIds.includes("gemini-3.1-flash-image")) { + if (ids.length >= limit) return null; + ids.push("gemini-3.1-flash-image"); + } + + const available = new Map>(); + for (const wireId of ids) { + const info = antigravityRecord(models[wireId]); + if (!info || available.has(wireId)) continue; + // Legacy compatibility aliases are deliberately routed to newer wire ids for saved + // selections. They are not safe as independently discovered picker rows. + if (ANTIGRAVITY_MODEL_ALIASES[wireId] && ANTIGRAVITY_MODEL_ALIASES[wireId] !== wireId) continue; + available.set(wireId, info); + } + + const out: AntigravityAvailableModel[] = []; + const seen = new Set(); + for (const [wireId, info] of available) { + const pickerId = ANTIGRAVITY_PICKER_MODEL_BY_WIRE_ID[wireId]; + const completePickerSet = pickerId !== undefined + && ANTIGRAVITY_WIRE_IDS_BY_PICKER_MODEL[pickerId]!.every(id => available.has(id)); + const id = completePickerSet ? pickerId! : wireId; + if (seen.has(id)) continue; + seen.add(id); + out.push({ + id, + ...(antigravityPositiveInteger(info.maxTokens) ? { contextWindow: antigravityPositiveInteger(info.maxTokens) } : {}), + inputModalities: info.supportsImages === true ? ["text", "image"] : ["text"], + }); + } + return out; +} + export function resolveAntigravityWireModelId(modelId: string): string { return ANTIGRAVITY_MODEL_ALIASES[modelId] ?? modelId; } diff --git a/src/providers/model-discovery-limits.ts b/src/providers/model-discovery-limits.ts new file mode 100644 index 000000000..7385b02a9 --- /dev/null +++ b/src/providers/model-discovery-limits.ts @@ -0,0 +1,16 @@ +/** Hard process-wide limits shared by all live model-discovery parsers. */ +export const MODEL_DISCOVERY_MAX_RESPONSE_BYTES = 4 * 1024 * 1024; +export const MODEL_DISCOVERY_MAX_MODELS = 2_000; +export const MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH = 1_024; + +const MODEL_DISCOVERY_MODEL_ID_CONTROL_CHARS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; + +/** Reject model IDs that cannot safely be published as callable catalog selectors. */ +export function isValidModelDiscoveryModelId(value: unknown): value is string { + if (typeof value !== "string") return false; + const normalized = value.trim(); + return Boolean(normalized) + && normalized === value + && normalized.length <= MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH + && !MODEL_DISCOVERY_MODEL_ID_CONTROL_CHARS.test(normalized); +} diff --git a/src/providers/model-discovery.ts b/src/providers/model-discovery.ts index d2bb5bbd8..534c0d00e 100644 --- a/src/providers/model-discovery.ts +++ b/src/providers/model-discovery.ts @@ -1,4 +1,16 @@ import type { OcxProviderConfig } from "../types"; +import { + isValidModelDiscoveryModelId, + MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH, + MODEL_DISCOVERY_MAX_MODELS, + MODEL_DISCOVERY_MAX_RESPONSE_BYTES, +} from "./model-discovery-limits"; +export { + isValidModelDiscoveryModelId, + MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH, + MODEL_DISCOVERY_MAX_MODELS, + MODEL_DISCOVERY_MAX_RESPONSE_BYTES, +} from "./model-discovery-limits"; import { getProviderRegistryEntry, providerMatchesRegistryTransport, @@ -9,13 +21,8 @@ import { type ProviderModelDiscoverySpec, } from "./registry"; -/** Hard process-wide limits. Registry entries may lower, but never raise, these ceilings. */ -export const MODEL_DISCOVERY_MAX_RESPONSE_BYTES = 4 * 1024 * 1024; -export const MODEL_DISCOVERY_MAX_MODELS = 2_000; -export const MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH = 1_024; const MODEL_DISCOVERY_MAX_FILTER_VALUES = 256; const MODEL_DISCOVERY_MAX_FILTER_STRING_LENGTH = 1_024; -const MODEL_DISCOVERY_MODEL_ID_CONTROL_CHARS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; export interface ResolvedProviderModelDiscovery { spec?: ProviderModelDiscoverySpec; @@ -343,16 +350,8 @@ export function extractProviderModelItems( return { ok: false, reason: "invalid_shape" }; } const id = (raw as { id?: unknown }).id; - if (typeof id !== "string") return { ok: false, reason: "invalid_shape" }; - const normalizedId = id.trim(); - if ( - !normalizedId - || normalizedId !== id - || normalizedId.length > MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH - || MODEL_DISCOVERY_MODEL_ID_CONTROL_CHARS.test(normalizedId) - ) { - return { ok: false, reason: "invalid_shape" }; - } + if (!isValidModelDiscoveryModelId(id)) return { ok: false, reason: "invalid_shape" }; + const normalizedId = id; const item = raw as ProviderModelsApiItem; if (!providerModelMatchesDiscoveryFilter(item, discovery.spec?.filter) || seen.has(normalizedId)) continue; seen.add(normalizedId); diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 656816253..8da2f91f8 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1,6 +1,6 @@ import type { CodexAccountMode, OcxProviderConfig } from "../types"; import { KIRO_MODELS, KIRO_MODEL_CONTEXT_WINDOWS, KIRO_MODEL_REASONING_EFFORTS } from "./kiro-models"; -import { ANTIGRAVITY_MODELS, ANTIGRAVITY_MODEL_CONTEXT_WINDOWS, ANTIGRAVITY_MODEL_EFFORTS } from "./antigravity-models"; +import { ANTIGRAVITY_MODELS, ANTIGRAVITY_MODEL_CONTEXT_WINDOWS, ANTIGRAVITY_MODEL_EFFORTS, ANTIGRAVITY_MODEL_INPUT_MODALITIES } from "./antigravity-models"; import type { ProviderBaseUrlChoice } from "./base-url-choices"; import { QWEN_CLOUD_BASE_URL_CHOICES, QWEN_CLOUD_TOKEN_PLAN_BASE_URL, @@ -1287,7 +1287,7 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ // 2026-07-10: defaultModel is frozen pending Vertex-specific Tier-2 evidence; Gemini API // evidence from ai.google.dev does not establish Vertex publisher availability. { id: "google-vertex", label: "Google Vertex AI", adapter: "google", baseUrl: "https://aiplatform.googleapis.com", authKind: "key", dashboardUrl: "https://console.cloud.google.com/vertex-ai", defaultModel: "gemini-3-pro", googleMode: "vertex", jawcodeBundle: "google", extraMetadataAliases: ["gemini-vertex"] }, - { id: "google-antigravity", label: "Google Antigravity", adapter: "google", baseUrl: "https://daily-cloudcode-pa.googleapis.com", authKind: "oauth", dashboardUrl: "https://antigravity.google", models: ANTIGRAVITY_MODELS, liveModels: false, defaultModel: "gemini-3.6-flash", modelContextWindows: ANTIGRAVITY_MODEL_CONTEXT_WINDOWS, modelReasoningEfforts: ANTIGRAVITY_MODEL_EFFORTS, googleMode: "cloud-code-assist", jawcodeBundle: "google", extraMetadataAliases: ["antigravity", "gemini-antigravity"] }, + { id: "google-antigravity", label: "Google Antigravity", adapter: "google", baseUrl: "https://daily-cloudcode-pa.googleapis.com", authKind: "oauth", dashboardUrl: "https://antigravity.google", models: ANTIGRAVITY_MODELS, liveModels: true, defaultModel: "gemini-3.6-flash", modelContextWindows: ANTIGRAVITY_MODEL_CONTEXT_WINDOWS, modelInputModalities: ANTIGRAVITY_MODEL_INPUT_MODALITIES, modelReasoningEfforts: ANTIGRAVITY_MODEL_EFFORTS, googleMode: "cloud-code-assist", jawcodeBundle: "google", extraMetadataAliases: ["antigravity", "gemini-antigravity"] }, { id: "azure-openai", label: "Azure OpenAI", adapter: "azure-openai", baseUrl: "https://{resource}.openai.azure.com/openai", authKind: "key", featured: true, dashboardUrl: "https://portal.azure.com" }, { id: "ollama", label: "Ollama (local)", adapter: "openai-chat", baseUrl: "http://localhost:11434/v1", authKind: "local", allowPrivateNetworkByDefault: true, allowBaseUrlOverride: true, featured: true, note: "Local — key usually blank" }, { id: "vllm", label: "vLLM (local)", adapter: "openai-chat", baseUrl: "http://localhost:8000/v1", authKind: "local", allowPrivateNetworkByDefault: true, allowBaseUrlOverride: true, featured: true, note: "Local — key usually blank" }, diff --git a/src/server/management/oauth-account-routes.ts b/src/server/management/oauth-account-routes.ts index f3db21c6a..3663d53a2 100644 --- a/src/server/management/oauth-account-routes.ts +++ b/src/server/management/oauth-account-routes.ts @@ -210,6 +210,10 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< await removeCredential(provider); reconcileLiveStateStores(); clearLoginState(provider); + const { clearModelCache } = await import("../../codex/model-cache"); + const { clearGatherRoutedModelsInflight } = await import("../../codex/catalog"); + clearModelCache(provider); + clearGatherRoutedModelsInflight(); // Drop cached/last-good quota rows tied to the removed credential. const { clearProviderQuotaCache, clearAccountQuotaCache } = await import("../../providers/quota"); clearProviderQuotaCache(); @@ -281,6 +285,10 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< const { resetAnthropicRoutingForManualSelection } = await import("../../oauth/anthropic-routing"); resetAnthropicRoutingForManualSelection(body.accountId); } + const { clearModelCache } = await import("../../codex/model-cache"); + const { clearGatherRoutedModelsInflight } = await import("../../codex/catalog"); + clearModelCache(provider); + clearGatherRoutedModelsInflight(); const { clearProviderQuotaCache } = await import("../../providers/quota"); clearProviderQuotaCache(); return jsonResponse({ ok: true, provider, activeAccountId: body.accountId }); @@ -404,6 +412,10 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< clearAnthropicSessionAffinityForAccount(id); } if (!getAccountSet(provider)) clearLoginState(provider); + const { clearModelCache } = await import("../../codex/model-cache"); + const { clearGatherRoutedModelsInflight } = await import("../../codex/catalog"); + clearModelCache(provider); + clearGatherRoutedModelsInflight(); const { clearProviderQuotaCache, clearAccountQuotaCache } = await import("../../providers/quota"); clearProviderQuotaCache(); clearAccountQuotaCache(provider); diff --git a/src/server/management/provider-routes.ts b/src/server/management/provider-routes.ts index 8a4a50d98..1fb4d5895 100644 --- a/src/server/management/provider-routes.ts +++ b/src/server/management/provider-routes.ts @@ -25,10 +25,11 @@ import { import { removeCredential } from "../../oauth/store"; import { providerDestinationResolvedError } from "../../lib/destination-policy"; import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; -import { ProviderOutboundPolicyError, providerOutboundGet, providerRedirectError } from "../../lib/provider-outbound"; +import { ProviderOutboundPolicyError, providerOutboundGet, providerOutboundPost, providerRedirectError } from "../../lib/provider-outbound"; +import { parseAntigravityAvailableModels } from "../../providers/antigravity-models"; import { enrichProviderFromCatalog, listKeyLoginProviders } from "../../oauth/key-providers"; import { deriveProviderPresets } from "../../providers/derive"; -import { providerCodexAccountMode, providerMatchesRegistryTransport } from "../../providers/registry"; +import { effectiveGoogleMode, providerCodexAccountMode, providerMatchesRegistryTransport } from "../../providers/registry"; import { extractModelEnvelopeRows, extractProviderModelItems, @@ -506,19 +507,33 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise undefined) + : undefined; + const apiKey = snapshot?.accessToken ?? await resolveModelsAuthToken(name, prov); if (prov.authMode === "oauth" && !apiKey) { return jsonResponse({ ok: false, latencyMs: 0, error: "static catalog only — upstream not verified (not logged in)" }); } - const { url: modelsUrl, headers } = buildModelsRequest(prov, apiKey, name); + const project = prov.project ?? snapshot?.projectId; + if (antigravity && !project) { + return jsonResponse({ ok: false, latencyMs: 0, error: "Antigravity project unavailable — re-run `ocx login google-antigravity`" }); + } + const { method, url: modelsUrl, headers } = buildModelsRequest(prov, apiKey, name); const discovery = resolveProviderModelDiscovery(name, prov); const started = Date.now(); try { - const res = await providerOutboundGet(name, prov, modelsUrl, { - headers, - signal: AbortSignal.timeout(8000), - }); + const res = method === "POST" + ? await providerOutboundPost(name, prov, modelsUrl, { + headers, + body: JSON.stringify({ project }), + signal: AbortSignal.timeout(8000), + }) + : await providerOutboundGet(name, prov, modelsUrl, { + headers, + signal: AbortSignal.timeout(8000), + }); const latencyMs = Date.now() - started; const redirectError = await providerRedirectError(res, modelsUrl); if (redirectError) { @@ -534,7 +549,7 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise : undefined; - const extracted = Array.isArray(bounded.value) || Array.isArray(record?.data) + const extracted = ccaModels + ? undefined + : Array.isArray(bounded.value) || Array.isArray(record?.data) ? extractProviderModelItems(bounded.value, discovery) : extractModelEnvelopeRows(bounded.value, discovery.maxModels, ["models"]); - if (!extracted.ok) { + if (extracted && !extracted.ok) { return jsonResponse({ ok: false, latencyMs, @@ -564,7 +585,7 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise { ...providerConfigSeed(entry), authMode: "key", apiKey: "test-token", + liveModels: false, }, }, })); diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index f3c01d199..17ad4283b 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -2232,9 +2232,10 @@ describe("Codex catalog routed normalization", () => { } }); - test("Google Antigravity uses its static registry catalog and suppresses stale discovery (#723)", async () => { + test("Google Antigravity honors an explicit static catalog and suppresses stale discovery", async () => { const providerName = "google-antigravity"; const provider = structuredClone(OAUTH_PROVIDERS[providerName].providerConfig); + provider.liveModels = false; const config = { port: 10100, defaultProvider: providerName, diff --git a/tests/config.test.ts b/tests/config.test.ts index cca41a4d6..0c1313b81 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -164,17 +164,15 @@ describe("opencodex config defaults", () => { writeConfig({ port: 12345, defaultProvider: "custom", - googleAntigravityStaticCatalogVersion: 99, + googleAntigravityStaticCatalogVersion: 2, providers: { custom: { adapter: "openai-chat", baseUrl: "https://example.test/v1" } }, }); - const degraded = loadConfig(); - expect(degraded.googleAntigravityStaticCatalogVersion).toBeUndefined(); - expect(degraded.providers.custom.baseUrl).toBe("https://example.test/v1"); + expect(loadConfig().googleAntigravityStaticCatalogVersion).toBe(2); expect(backupNames()).toEqual([]); expect(validateConfigCandidate({ ...getDefaultConfig(), - googleAntigravityStaticCatalogVersion: 99, + googleAntigravityStaticCatalogVersion: 3, })).toMatchObject({ ok: false, error: expect.stringContaining("googleAntigravityStaticCatalogVersion"), diff --git a/tests/cursor-hardening.test.ts b/tests/cursor-hardening.test.ts index 4de7bc0ab..c7bf93b95 100644 --- a/tests/cursor-hardening.test.ts +++ b/tests/cursor-hardening.test.ts @@ -109,6 +109,44 @@ describe("Cursor live-model discovery hardening", () => { } }); + test("does not warn when a failed Cursor discovery belongs to a cleared generation", async () => { + const provider = "cursor-discovery-stale-warning"; + const warning = spyOn(console, "warn").mockImplementation(() => {}); + let release!: () => void; + const started = new Promise(resolve => { release = resolve; }); + let stream!: http2.ServerHttp2Stream; + try { + await withDiscoveryServer(candidate => { + stream = candidate; + release(); + }, async baseUrl => { + const pending = gatherRoutedModels({ + providers: { + [provider]: { + adapter: "cursor", + baseUrl, + apiKey: "test-token", + models: ["auto"], + }, + }, + }); + await started; + clearModelCache(provider); + stream.respond({ ":status": 401, "content-type": "application/proto" }); + stream.end(); + await pending; + }); + + expect(warning.mock.calls.some(args => String(args[0]).includes( + `Cursor model discovery for "${provider}" failed`, + ))).toBe(false); + expect(getProviderDiscoveryStatus(provider)).toBeUndefined(); + } finally { + warning.mockRestore(); + clearModelCache(provider); + } + }); + test("classifies non-auth HTTP failures", async () => { const result = await withDiscoveryServer(respond(503), baseUrl => fetchCursorUsableModels({ apiKey: "test-token", baseUrl })); diff --git a/tests/google-antigravity-wire.test.ts b/tests/google-antigravity-wire.test.ts index 3c1457abd..5bab83070 100644 --- a/tests/google-antigravity-wire.test.ts +++ b/tests/google-antigravity-wire.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from "bun:test"; import { createGoogleAdapter as createGoogleAdapterProduction } from "../src/adapters/google"; import { antigravitySessionId, isLikelyRealThoughtSignature } from "../src/adapters/google-antigravity-wire"; -import { ANTIGRAVITY_MODELS, ANTIGRAVITY_MODEL_EFFORTS, canonicalAntigravityUsageModel } from "../src/providers/antigravity-models"; +import { ANTIGRAVITY_MODELS, ANTIGRAVITY_MODEL_EFFORTS, canonicalAntigravityUsageModel, parseAntigravityAvailableModels } from "../src/providers/antigravity-models"; +import { MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH, MODEL_DISCOVERY_MAX_MODELS } from "../src/providers/model-discovery"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../src/types"; import { withTestTranslatorBudget } from "./helpers/translator-budget"; @@ -118,6 +119,81 @@ describe("antigravity CCA envelope", () => { } }); + test("collapses a complete CCA Gemini tier set but retains partial sets as wire IDs", () => { + const payload = (modelIds: string[]) => ({ + models: Object.fromEntries(modelIds.map(id => [id, { maxTokens: 1_048_576 }])), + agentModelSorts: [{ groups: [{ modelIds }] }], + }); + + expect(parseAntigravityAvailableModels(payload([ + "gemini-3.6-flash-low", + "gemini-3.6-flash-medium", + "gemini-3.6-flash-high", + ]))?.map(model => model.id)).toEqual(["gemini-3.6-flash"]); + expect(parseAntigravityAvailableModels(payload([ + "gemini-3.6-flash-low", + "gemini-3.6-flash-high", + ]))?.map(model => model.id)).toEqual([ + "gemini-3.6-flash-low", + "gemini-3.6-flash-high", + ]); + }); + + test("rejects malformed and oversized CCA agent-model lists", () => { + const payload = (modelIds: unknown[]) => ({ + models: Object.fromEntries(modelIds.map(id => [String(id), { maxTokens: 1_048_576 }])), + agentModelSorts: [{ groups: [{ modelIds }] }], + }); + + for (const invalidId of [" ", "bad\u0000id", "x".repeat(MODEL_DISCOVERY_MAX_MODEL_ID_LENGTH + 1)]) { + expect(parseAntigravityAvailableModels(payload([invalidId]))).toBeNull(); + } + expect(parseAntigravityAvailableModels({ + models: {}, + agentModelSorts: [{ groups: [{ + modelIds: Array.from({ length: MODEL_DISCOVERY_MAX_MODELS + 1 }, (_, index) => `model-${index}`), + }] }], + })).toBeNull(); + }); + + test("rejects malformed CCA agent-model containers and missing agent metadata", () => { + expect(parseAntigravityAvailableModels({ + models: {}, + agentModelSorts: [{}], + })).toBeNull(); + expect(parseAntigravityAvailableModels({ + models: {}, + agentModelSorts: [{ groups: {} }], + })).toBeNull(); + expect(parseAntigravityAvailableModels({ + models: {}, + agentModelSorts: [{ groups: [{ modelIds: {} }] }], + })).toBeNull(); + expect(parseAntigravityAvailableModels({ + models: {}, + agentModelSorts: [{ groups: [{ modelIds: ["agent-model"] }] }], + })).toBeNull(); + }); + + test("normalizes untrusted CCA model limits before publishing a catalog", () => { + const oversized = Array.from( + { length: MODEL_DISCOVERY_MAX_MODELS + 1 }, + (_, index) => `model-${index}`, + ); + const payload = { + models: Object.fromEntries(oversized.map(id => [id, { maxTokens: 1_048_576 }])), + agentModelSorts: [{ groups: [{ modelIds: oversized }] }], + }; + for (const limit of [Number.NaN, Infinity, MODEL_DISCOVERY_MAX_MODELS + 1]) { + expect(parseAntigravityAvailableModels(payload, limit)).toBeNull(); + } + expect(parseAntigravityAvailableModels({ + models: { "agent-model": { maxTokens: 1_048_576 } }, + agentModelSorts: [{ groups: [{ modelIds: ["agent-model"] }] }], + imageGenerationModelIds: ["gemini-3.1-flash-image"], + }, 1)).toBeNull(); + }); + test("throws when no project id is available", async () => { const noProj = { ...provider, project: undefined } as OcxProviderConfig; await expect(createGoogleAdapter(noProj).buildRequest(parsed())).rejects.toThrow(/project id/); diff --git a/tests/google-models-listing.test.ts b/tests/google-models-listing.test.ts index 0a26fc3c7..419cc7655 100644 --- a/tests/google-models-listing.test.ts +++ b/tests/google-models-listing.test.ts @@ -1,5 +1,8 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test"; -import { gatherRoutedModels as gatherRoutedModelsDirect } from "../src/codex/catalog"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildCatalogEntries, gatherRoutedModels as gatherRoutedModelsDirect } from "../src/codex/catalog"; import { buildModelsRequest } from "../src/oauth"; import { clearModelCache, getStaleCached } from "../src/codex/model-cache"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; @@ -10,10 +13,13 @@ const gatherRoutedModels: typeof gatherRoutedModelsDirect = (config, options) => gatherRoutedModelsDirect(withStubbedProviderFetch(config), options); const originalFetch = globalThis.fetch; +const originalOpencodexHome = process.env.OPENCODEX_HOME; afterEach(() => { globalThis.fetch = originalFetch; clearModelCache(); + if (originalOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalOpencodexHome; }); function configWith(name: string, prov: Partial): OcxConfig { @@ -38,13 +44,15 @@ describe("buildModelsRequest google routing", () => { expect(headers["x-goog-api-key"]).toBe("gk-123"); }); - test("an explicit Antigravity live-discovery override keeps Authorization: Bearer", () => { - // Static discovery is the preset default. If a user explicitly opts into the generic probe, - // a saved config may still omit googleMode — the registry's cloud-code-assist mode must win. + test("Antigravity uses its authenticated CCA model-discovery RPC", () => { + // A saved config may omit googleMode — the registry's cloud-code-assist mode must win. const prov = { adapter: "google", authMode: "oauth", baseUrl: "https://daily-cloudcode-pa.googleapis.com", liveModels: true } as OcxProviderConfig; - const { url, headers } = buildModelsRequest(prov, "oauth-token", "google-antigravity"); - expect(url).toBe("https://daily-cloudcode-pa.googleapis.com/models"); + const { method, url, headers } = buildModelsRequest(prov, "oauth-token", "google-antigravity"); + expect(method).toBe("POST"); + expect(url).toBe("https://daily-cloudcode-pa.googleapis.com/v1internal:fetchAvailableModels"); expect(headers["Authorization"]).toBe("Bearer oauth-token"); + expect(headers["Content-Type"]).toBe("application/json"); + expect(headers.Accept).toBe("application/json"); expect(headers["x-goog-api-key"]).toBeUndefined(); }); @@ -55,6 +63,175 @@ describe("buildModelsRequest google routing", () => { }); }); +describe("Antigravity live model discovery", () => { + test("uses the CCA agent list and applies CCA metadata", async () => { + const home = mkdtempSync(join(tmpdir(), "ocx-antigravity-discovery-")); + process.env.OPENCODEX_HOME = home; + writeFileSync(join(home, "auth.json"), JSON.stringify({ + "google-antigravity": { + activeAccountId: "active", + accounts: [{ + id: "active", + credential: { + access: "access-token", + refresh: "refresh-token", + expires: Date.now() + 3_600_000, + projectId: "project-id", + }, + }], + }, + })); + const seen: { url: string; init?: RequestInit }[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(input), init }); + return Response.json({ + models: { + "gemini-3.6-flash-low": { maxTokens: 1_048_576, supportsImages: true, supportsThinking: true, thinkingBudget: 1000 }, + "gemini-3.6-flash-high": { maxTokens: 1_048_576, supportsImages: true, supportsThinking: true, thinkingBudget: 10000 }, + "future-agent-model": { maxTokens: 333_333, supportsImages: false, supportsThinking: true, thinkingBudget: 7777 }, + "gemini-3.1-flash-image": { maxTokens: 555_555, supportsImages: true }, + "non-agent-command-model": { maxTokens: 222_222 }, + "tab-only-model": { maxTokens: 32_768 }, + }, + agentModelSorts: [{ groups: [{ modelIds: [ + "future-agent-model", "gemini-3.6-flash-low", "gemini-3.6-flash-high", + ] }] }], + imageGenerationModelIds: ["gemini-3.1-flash-image"], + tabModelIds: ["tab-only-model"], + commandModelIds: ["non-agent-command-model"], + }); + }) as typeof fetch; + + try { + const models = await gatherRoutedModels(configWith("google-antigravity", { + adapter: "google", + authMode: "oauth", + baseUrl: "https://daily-cloudcode-pa.googleapis.com", + project: "configured-project", + liveModels: true, + models: ["configured-only"], + })); + const live = models.filter(model => model.provider === "google-antigravity"); + + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe("https://daily-cloudcode-pa.googleapis.com/v1internal:fetchAvailableModels"); + expect(seen[0]?.init?.method).toBe("POST"); + expect((seen[0]?.init?.headers as Record).Authorization).toBe("Bearer access-token"); + expect(JSON.parse(String(seen[0]?.init?.body))).toEqual({ project: "configured-project" }); + expect(live.map(model => model.id).sort()).toEqual([ + "future-agent-model", + "gemini-3.1-flash-image", + "gemini-3.6-flash-high", + "gemini-3.6-flash-low", + ]); + expect(live.find(model => model.id === "gemini-3.6-flash-low")).toMatchObject({ + contextWindow: 1_048_576, + inputModalities: ["text", "image"], + reasoningEfforts: [], + }); + expect(live.find(model => model.id === "future-agent-model")).toMatchObject({ + contextWindow: 333_333, + inputModalities: ["text"], + reasoningEfforts: [], + }); + expect(live.map(model => model.id)).not.toContain("tab-only-model"); + expect(live.map(model => model.id)).not.toContain("non-agent-command-model"); + + const catalog = buildCatalogEntries(null, [], live); + const flashLow = catalog.find(entry => entry.slug === "google-antigravity/gemini-3.6-flash-low"); + const flashHigh = catalog.find(entry => entry.slug === "google-antigravity/gemini-3.6-flash-high"); + const future = catalog.find(entry => entry.slug === "google-antigravity/future-agent-model"); + expect(flashLow).toMatchObject({ + context_window: 1_048_576, + max_context_window: 1_048_576, + auto_compact_token_limit: 943_718, + input_modalities: ["text", "image"], + }); + expect(flashLow).not.toHaveProperty("default_reasoning_level"); + expect(flashLow?.supported_reasoning_levels).toEqual([]); + expect(flashHigh).toBeDefined(); + expect(catalog.map(entry => entry.slug)).not.toContain("google-antigravity/gemini-3.6-flash"); + expect(catalog.map(entry => entry.slug)).not.toContain("google-antigravity/gemini-3.6-flash-medium"); + expect(future).toMatchObject({ + context_window: 333_333, + max_context_window: 333_333, + auto_compact_token_limit: 299_999, + input_modalities: ["text"], + }); + expect(future).not.toHaveProperty("default_reasoning_level"); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test("degrades malformed CCA agent IDs to the configured static catalog", async () => { + const home = mkdtempSync(join(tmpdir(), "ocx-antigravity-malformed-discovery-")); + process.env.OPENCODEX_HOME = home; + writeFileSync(join(home, "auth.json"), JSON.stringify({ + "google-antigravity": { + activeAccountId: "active", + accounts: [{ + id: "active", + credential: { + access: "access-token", + refresh: "refresh-token", + expires: Date.now() + 3_600_000, + projectId: "project-id", + }, + }], + }, + })); + const warning = spyOn(console, "warn").mockImplementation(() => {}); + globalThis.fetch = (async () => Response.json({ + models: { "bad\u0000model": { maxTokens: 1_048_576 } }, + agentModelSorts: [{ groups: [{ modelIds: ["bad\u0000model"] }] }], + })) as typeof fetch; + + try { + const models = await gatherRoutedModels(configWith("google-antigravity", { + adapter: "google", + authMode: "oauth", + baseUrl: "https://daily-cloudcode-pa.googleapis.com", + liveModels: true, + models: ["configured-only"], + })); + + expect(models.filter(model => model.provider === "google-antigravity").map(model => model.id)) + .toEqual(["configured-only"]); + expect(getStaleCached("google-antigravity")).toBeNull(); + } finally { + warning.mockRestore(); + rmSync(home, { recursive: true, force: true }); + } + }); + + test("uses the configured key for a custom CCA provider", async () => { + const seen: { headers: Record }[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ headers: (init?.headers ?? {}) as Record }); + return Response.json({ + models: { "custom-agent-model": { maxTokens: 1_048_576 } }, + agentModelSorts: [{ groups: [{ modelIds: ["custom-agent-model"] }] }], + }); + }) as typeof fetch; + + const models = await gatherRoutedModels(configWith("custom-cca", { + adapter: "google", + authMode: "key", + apiKey: "custom-cca-key", + baseUrl: "https://daily-cloudcode-pa.googleapis.com", + googleMode: "cloud-code-assist", + project: "configured-project", + liveModels: true, + })); + + expect(seen).toHaveLength(1); + expect(seen[0]?.headers.Authorization).toBe("Bearer custom-cca-key"); + expect(models.filter(model => model.provider === "custom-cca").map(model => model.id)) + .toEqual(["custom-agent-model"]); + }); +}); + describe("buildModelsRequest anthropic routing", () => { test("normalizes a /v1 baseUrl and keeps the Anthropic models path singular", () => { const prov = { diff --git a/tests/helpers/provider-registry-discovery.ts b/tests/helpers/provider-registry-discovery.ts index 2d1e28e65..b98db5c68 100644 --- a/tests/helpers/provider-registry-discovery.ts +++ b/tests/helpers/provider-registry-discovery.ts @@ -16,7 +16,7 @@ export async function withRegistryDiscovery( if (!entry) throw new Error(`missing ${providerId} registry entry`); const originalDiscovery = entry.modelDiscovery; const originalPreserveCustomDestination = entry.preserveCustomDestination; - clearModelCache(providerId); + clearModelCache(providerId, "eviction"); entry.modelDiscovery = spec; if (overrides.preserveCustomDestination !== undefined) { entry.preserveCustomDestination = overrides.preserveCustomDestination; @@ -28,6 +28,6 @@ export async function withRegistryDiscovery( else entry.modelDiscovery = originalDiscovery; if (originalPreserveCustomDestination === undefined) delete entry.preserveCustomDestination; else entry.preserveCustomDestination = originalPreserveCustomDestination; - clearModelCache(providerId); + clearModelCache(providerId, "eviction"); } } diff --git a/tests/model-cache.test.ts b/tests/model-cache.test.ts new file mode 100644 index 000000000..d3b30f372 --- /dev/null +++ b/tests/model-cache.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + captureModelCacheGeneration, + clearModelCache, + getStaleCached, + reconcileModelCacheProviders, + setCached, +} from "../src/codex/model-cache"; + +const provider = "removed-provider-generation"; + +afterEach(() => clearModelCache(provider)); + +describe("model-cache provider reconciliation", () => { + test.each([ + ["provider", () => clearModelCache(provider, "eviction")], + ["global", () => clearModelCache(undefined, "eviction")], + ])("%s eviction keeps an in-flight discovery authorized", (_scope, evict) => { + const captured = captureModelCacheGeneration(provider); + + evict(); + + expect(setCached(provider, [{ provider, id: "late-model" }], Date.now(), captured)).toBe(true); + expect(getStaleCached(provider)).toEqual([{ provider, id: "late-model" }]); + }); + + test.each([ + ["provider", () => clearModelCache(provider)], + ["global", () => clearModelCache()], + ])("%s authority change rejects an in-flight discovery", (_scope, revokeAuthority) => { + const captured = captureModelCacheGeneration(provider); + + revokeAuthority(); + + expect(setCached(provider, [{ provider, id: "late-model" }], Date.now(), captured)).toBe(false); + expect(getStaleCached(provider)).toBeNull(); + }); + + test("rejects an in-flight write for a provider removed before it has a cache entry", () => { + const captured = captureModelCacheGeneration(provider); + + expect(reconcileModelCacheProviders(new Set(), Date.now())).toBe(1); + expect(setCached(provider, [{ provider, id: "late-model" }], Date.now(), captured)).toBe(false); + expect(getStaleCached(provider)).toBeNull(); + }); +}); diff --git a/tests/oauth-accounts-api.test.ts b/tests/oauth-accounts-api.test.ts index 7d50b81dc..280e8cfb3 100644 --- a/tests/oauth-accounts-api.test.ts +++ b/tests/oauth-accounts-api.test.ts @@ -5,12 +5,19 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../src/config"; import { startServer } from "../src/server"; +import { gatherRoutedModels as gatherRoutedModelsDirect } from "../src/codex/catalog"; +import { clearModelCache, getStaleCached, setCached } from "../src/codex/model-cache"; import type { OcxConfig } from "../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; +import { withStubbedProviderFetch } from "./helpers/catalog-provider-fetch"; let testDir = ""; let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; +const originalFetch = globalThis.fetch; + +const gatherRoutedModels: typeof gatherRoutedModelsDirect = config => + gatherRoutedModelsDirect(withStubbedProviderFetch(config)); function baseConfig(): OcxConfig { return { @@ -45,6 +52,8 @@ beforeEach(() => { }); afterEach(() => { + globalThis.fetch = originalFetch; + clearModelCache("google-antigravity"); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; isolatedCodexHome?.restore(); @@ -175,6 +184,101 @@ describe("multiauth accounts API", () => { } }); + test("switching an Antigravity account clears its account-scoped live-model cache", async () => { + writeFileSync(join(testDir, "auth.json"), JSON.stringify({ + "google-antigravity": { + activeAccountId: "antigravity-a", + accounts: [ + { id: "antigravity-a", credential: { access: "a", refresh: "ra", expires: 9999999999999, projectId: "project-a" } }, + { id: "antigravity-b", credential: { access: "b", refresh: "rb", expires: 9999999999999, projectId: "project-b" } }, + ], + }, + }), { mode: 0o600 }); + const server = startServer(0); + try { + setCached("google-antigravity", [{ provider: "google-antigravity", id: "account-a-only-model" }]); + const response = await fetch(new URL("/api/oauth/accounts/active", server.url), { + method: "PUT", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ provider: "google-antigravity", accountId: "antigravity-b" }), + }); + + expect(response.status).toBe(200); + expect(getStaleCached("google-antigravity")).toBeNull(); + } finally { + await server.stop(true); + } + }); + + test("switching an Antigravity account discards an in-flight prior-account discovery", async () => { + writeFileSync(join(testDir, "auth.json"), JSON.stringify({ + "google-antigravity": { + activeAccountId: "antigravity-a", + accounts: [ + { id: "antigravity-a", credential: { access: "account-a-token", refresh: "ra", expires: 9999999999999, projectId: "project-a" } }, + { id: "antigravity-b", credential: { access: "account-b-token", refresh: "rb", expires: 9999999999999, projectId: "project-b" } }, + ], + }, + }), { mode: 0o600 }); + const config = { + providers: { + "google-antigravity": { + adapter: "google", + authMode: "oauth", + baseUrl: "https://daily-cloudcode-pa.googleapis.com", + liveModels: true, + }, + }, + } as unknown as OcxConfig; + let releaseAccountA!: () => void; + const accountAStarted = new Promise(resolve => { + releaseAccountA = resolve; + }); + let accountAFetchStarted!: () => void; + const accountAFetchObserved = new Promise(resolve => { + accountAFetchStarted = resolve; + }); + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + if (String(input).includes("/api/")) return originalFetch(input, init); + const authorization = new Headers(init?.headers).get("Authorization"); + expect(authorization).toBe("Bearer account-a-token"); + accountAFetchStarted(); + await accountAStarted; + return Response.json({ + models: { "account-a-model": { maxTokens: 16_384 } }, + agentModelSorts: [{ groups: [{ modelIds: ["account-a-model"] }] }], + }); + }) as typeof fetch; + + const server = startServer(0); + try { + const first = gatherRoutedModels(config); + await accountAFetchObserved; + const switched = await fetch(new URL("/api/oauth/accounts/active", server.url), { + method: "PUT", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ provider: "google-antigravity", accountId: "antigravity-b" }), + }); + expect(switched.status).toBe(200); + releaseAccountA(); + await first; + expect(getStaleCached("google-antigravity")).toBeNull(); + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + if (String(input).includes("/api/")) return originalFetch(input, init); + expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer account-b-token"); + expect(JSON.parse(String(init?.body))).toEqual({ project: "project-b" }); + return Response.json({ + models: { "account-b-model": { maxTokens: 32_768 } }, + agentModelSorts: [{ groups: [{ modelIds: ["account-b-model"] }] }], + }); + }) as typeof fetch; + const second = await gatherRoutedModels(config); + expect(second.map(model => model.id)).toEqual(["account-b-model"]); + expect(getStaleCached("google-antigravity")?.map(model => model.id)).toEqual(["account-b-model"]); + } finally { + await server.stop(true); + } + }); + test("DELETE removes one account; active removal promotes the other", async () => { const server = startServer(0); try { diff --git a/tests/oauth-provider-reconcile.test.ts b/tests/oauth-provider-reconcile.test.ts index 1ee083065..611a4e969 100644 --- a/tests/oauth-provider-reconcile.test.ts +++ b/tests/oauth-provider-reconcile.test.ts @@ -17,7 +17,7 @@ afterEach(() => { }); describe("OAuth provider reconciliation", () => { - test("migrates a saved Antigravity 3.5 preset without touching credentials or user fields", async () => { + test("refreshes a saved Antigravity 3.5 preset without touching credentials or user fields", async () => { const home = mkdtempSync(join(tmpdir(), "ocx-gemini-36-reconcile-")); homes.push(home); process.env.OPENCODEX_HOME = home; @@ -41,8 +41,6 @@ describe("OAuth provider reconciliation", () => { modelContextWindows: { "gemini-3.5-flash-low": 1_048_576 }, project: "config-project-sentinel", note: "user-owned-note", - // This is deliberately ambiguous: old Provider Settings saves and manual edits - // persisted the same value, so the versioned migration normalizes both once. liveModels: true, }, }, @@ -64,8 +62,7 @@ describe("OAuth provider reconciliation", () => { expect(provider.models).not.toContain("gemini-3.6-flash-medium"); expect(provider.models).not.toContain("gemini-3.6-flash-high"); expect(provider.modelContextWindows?.["gemini-3.6-flash"]).toBe(1_048_576); - expect(provider.liveModels).toBe(false); - expect(config.googleAntigravityStaticCatalogVersion).toBe(1); + expect(provider.liveModels).toBe(true); expect(provider.project).toBe("config-project-sentinel"); expect(provider.note).toBe("user-owned-note"); expect(getCredential("google-antigravity")).toMatchObject({ @@ -76,12 +73,11 @@ describe("OAuth provider reconciliation", () => { const persisted = loadConfig(); expect(persisted.providers["google-antigravity"]?.defaultModel).toBe("gemini-3.6-flash"); - expect(persisted.providers["google-antigravity"]?.liveModels).toBe(false); - expect(persisted.googleAntigravityStaticCatalogVersion).toBe(1); + expect(persisted.providers["google-antigravity"]?.liveModels).toBe(true); expect(reconcileOAuthProviders(config)).toBe(false); }); - test("preserves an explicit Antigravity liveModels override during reconcile and re-login", () => { + test("migrates the version-1 canonical Antigravity static row to live discovery", () => { const config = { port: 10100, defaultProvider: "google-antigravity", @@ -89,20 +85,38 @@ describe("OAuth provider reconciliation", () => { providers: { "google-antigravity": { ...structuredClone(OAUTH_PROVIDERS["google-antigravity"].providerConfig), - liveModels: true, + liveModels: false, }, }, } satisfies OcxConfig; - expect(reconcileOAuthProviders(config)).toBe(false); + expect(reconcileOAuthProviders(config)).toBe(true); expect(config.providers["google-antigravity"].liveModels).toBe(true); + expect(config.googleAntigravityStaticCatalogVersion).toBe(2); upsertOAuthProvider(config, "google-antigravity"); expect(config.providers["google-antigravity"].liveModels).toBe(true); expect(config.providers["google-antigravity"].models).toHaveLength(6); }); - test("normalizes ambiguous pre-marker Antigravity rows even when authMode is omitted or non-OAuth", () => { + test("preserves an explicit Antigravity static opt-out without the legacy migration marker", () => { + const config = { + port: 10100, + defaultProvider: "google-antigravity", + providers: { + "google-antigravity": { + ...structuredClone(OAUTH_PROVIDERS["google-antigravity"].providerConfig), + liveModels: false, + }, + }, + } satisfies OcxConfig; + + expect(reconcileOAuthProviders(config)).toBe(false); + upsertOAuthProvider(config, "google-antigravity"); + expect(config.providers["google-antigravity"].liveModels).toBe(false); + }); + + test("preserves explicit Antigravity live discovery when authMode is omitted or non-OAuth", () => { const home = mkdtempSync(join(tmpdir(), "ocx-antigravity-authmode-reconcile-")); homes.push(home); process.env.OPENCODEX_HOME = home; @@ -123,17 +137,16 @@ describe("OAuth provider reconciliation", () => { providers: { "google-antigravity": provider }, } satisfies OcxConfig; - expect(reconcileOAuthProviders(config)).toBe(true); - expect(config.googleAntigravityStaticCatalogVersion).toBe(1); + expect(reconcileOAuthProviders(config)).toBe(false); const migrated = config.providers["google-antigravity"]; - expect(migrated.liveModels).toBe(false); - expect(migrated.defaultModel).toBe(preset.defaultModel); - expect(migrated.models).toEqual(preset.models); + expect(migrated.liveModels).toBe(true); + expect(migrated.defaultModel).toBe("gemini-3.5-flash-low"); + expect(migrated.models).toEqual(["gemini-3.5-flash-low", "gemini-3.5-flash-high"]); expect(migrated.authMode).toBe(authMode); } }); - test("normalizes ambiguous pre-marker true during re-login, then preserves later overrides", () => { + test("preserves Antigravity live discovery during re-login", () => { const config = { port: 10100, defaultProvider: "google-antigravity", @@ -146,8 +159,7 @@ describe("OAuth provider reconciliation", () => { } satisfies OcxConfig; upsertOAuthProvider(config, "google-antigravity"); - expect(config.googleAntigravityStaticCatalogVersion).toBe(1); - expect(config.providers["google-antigravity"].liveModels).toBe(false); + expect(config.providers["google-antigravity"].liveModels).toBe(true); config.providers["google-antigravity"].liveModels = true; config.providers["google-antigravity"].authMode = "key"; diff --git a/tests/provider-connection-test.test.ts b/tests/provider-connection-test.test.ts index 2ada6a8b9..c33f1ee17 100644 --- a/tests/provider-connection-test.test.ts +++ b/tests/provider-connection-test.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { handleManagementAPI } from "../src/server/management-api"; import { saveConfig } from "../src/config"; import { OAUTH_PROVIDERS } from "../src/oauth"; +import { saveCredential } from "../src/oauth/store"; import { PROVIDER_REGISTRY } from "../src/providers/registry"; import type { OcxConfig } from "../src/types"; import { withRegistryDiscovery } from "./helpers/provider-registry-discovery"; @@ -104,7 +105,10 @@ describe("POST /api/providers/test (WP040 connectivity probe)", () => { throw new Error("static Antigravity catalog must not probe upstream"); }) as typeof fetch; const config = baseConfig({ - "google-antigravity": structuredClone(OAUTH_PROVIDERS["google-antigravity"].providerConfig), + "google-antigravity": { + ...structuredClone(OAUTH_PROVIDERS["google-antigravity"].providerConfig), + liveModels: false, + }, }); const { body } = await probe(config, "google-antigravity"); @@ -113,6 +117,42 @@ describe("POST /api/providers/test (WP040 connectivity probe)", () => { expect(fetches).toBe(0); }); + test("Google Antigravity probes its CCA agent-model RPC", async () => { + const seen: { url: string; init?: RequestInit }[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(input), init }); + return Response.json({ + models: { + "any-agent-model": { maxTokens: 123_456 }, + "not-an-agent-model": { maxTokens: 65_536 }, + }, + agentModelSorts: [{ groups: [{ modelIds: ["any-agent-model"] }] }], + tabModelIds: ["not-an-agent-model"], + }); + }) as typeof fetch; + await saveCredential("google-antigravity", { + access: "test-access-token", + refresh: "test-refresh-token", + expires: Date.now() + 3_600_000, + projectId: "test-project-id", + }); + const config = baseConfig({ + "google-antigravity": { + ...structuredClone(OAUTH_PROVIDERS["google-antigravity"].providerConfig), + project: "configured-project", + }, + }); + + const { body } = await probe(config, "google-antigravity"); + + expect(body).toMatchObject({ ok: true, models: 1 }); + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe("https://daily-cloudcode-pa.googleapis.com/v1internal:fetchAvailableModels"); + expect(seen[0]?.init?.method).toBe("POST"); + expect((seen[0]?.init?.headers as Record).Authorization).toBe("Bearer test-access-token"); + expect(JSON.parse(String(seen[0]?.init?.body))).toEqual({ project: "configured-project" }); + }); + test("a fake key gets the upstream rejection, not a catalog-presence pass", async () => { globalThis.fetch = (async () => new Response("unauthorized", { status: 401 })) as typeof fetch; const config = baseConfig({ diff --git a/tests/provider-outbound.test.ts b/tests/provider-outbound.test.ts index 2fc871208..8f5e944f2 100644 --- a/tests/provider-outbound.test.ts +++ b/tests/provider-outbound.test.ts @@ -21,9 +21,9 @@ function directDependencies( options?: { privateNetwork?: boolean; address?: string }, ): { dependencies: ProviderOutboundDependencies; - captured: { address?: string; rejectUnauthorized?: boolean; authorization?: string }; + captured: { address?: string; rejectUnauthorized?: boolean; authorization?: string; body?: string }; } { - const captured: { address?: string; rejectUnauthorized?: boolean; authorization?: string } = {}; + const captured: { address?: string; rejectUnauthorized?: boolean; authorization?: string; body?: string } = {}; const address = options?.address ?? "93.184.216.34"; return { captured, @@ -39,6 +39,13 @@ function directDependencies( captured.authorization = new Headers(requestOptions?.headers).get("authorization") ?? undefined; return response; }), + pinnedPost: mock(async (_url, pinned, body, _signal, requestOptions) => { + captured.address = pinned.address; + captured.rejectUnauthorized = requestOptions?.rejectUnauthorized; + captured.authorization = new Headers(requestOptions?.headers).get("authorization") ?? undefined; + captured.body = body; + return response; + }), }, }; } @@ -238,3 +245,71 @@ describe("provider outbound GET transport", () => { } }, 15_000); }); + +describe("provider outbound POST transport", () => { + test("direct HTTPS posts only to the validated address with its credential and body", async () => { + for (const key of proxyKeys) delete process.env[key]; + const { providerOutboundPost } = await import("../src/lib/provider-outbound"); + const { dependencies, captured } = directDependencies(new Response('{"models":{}}', { + status: 200, + headers: { "content-type": "application/json" }, + })); + const body = JSON.stringify({ project: "test-project" }); + + const response = await providerOutboundPost( + "google-antigravity", + { baseUrl: "https://provider.example" }, + "https://provider.example/v1internal:fetchAvailableModels", + { headers: { authorization: "Bearer test-token" }, body }, + dependencies, + ); + + expect(await response.json()).toEqual({ models: {} }); + expect(captured).toEqual({ + address: "93.184.216.34", + rejectUnauthorized: true, + authorization: "Bearer test-token", + body, + }); + }); + + test("blocks an unsafe POST destination before invoking a caller-owned executor", async () => { + const { providerOutboundPost, ProviderOutboundPolicyError } = await import("../src/lib/provider-outbound"); + let calls = 0; + const provider = { + baseUrl: "https://provider.example", + fetch: (async () => { + calls += 1; + return new Response("{}"); + }) as typeof fetch, + }; + + await expect(providerOutboundPost( + "google-antigravity", + provider, + "https://169.254.169.254/v1internal:fetchAvailableModels", + { headers: { authorization: "Bearer test-token" }, body: '{"project":"test-project"}' }, + )).rejects.toThrow(ProviderOutboundPolicyError); + expect(calls).toBe(0); + }); + + test("requires HTTPS before invoking a caller-owned executor", async () => { + const { providerOutboundPost, ProviderOutboundPolicyError } = await import("../src/lib/provider-outbound"); + let calls = 0; + const provider = { + baseUrl: "https://provider.example", + fetch: (async () => { + calls += 1; + return new Response("{}"); + }) as typeof fetch, + }; + + await expect(providerOutboundPost( + "google-antigravity", + provider, + "http://93.184.216.34/v1internal:fetchAvailableModels", + { headers: { authorization: "Bearer test-token" }, body: '{"project":"test-project"}' }, + )).rejects.toThrow(ProviderOutboundPolicyError); + expect(calls).toBe(0); + }); +}); diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 0b24e63a4..5f205e8a3 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -640,9 +640,9 @@ describe("provider registry parity", () => { expect(OAUTH_PROVIDERS.xai.providerConfig.modelReasoningEfforts?.["grok-4.5"]).toEqual(["low", "medium", "high"]); expect(OAUTH_PROVIDERS.xai.providerConfig.noVisionModels).toContain("grok-build-0.1"); const antigravityRegistry = PROVIDER_REGISTRY.find(entry => entry.id === "google-antigravity"); - expect(antigravityRegistry?.liveModels).toBe(false); - expect(providerConfigSeed(antigravityRegistry!).liveModels).toBe(false); - expect(OAUTH_PROVIDERS["google-antigravity"].providerConfig.liveModels).toBe(false); + expect(antigravityRegistry?.liveModels).toBe(true); + expect(providerConfigSeed(antigravityRegistry!).liveModels).toBe(true); + expect(OAUTH_PROVIDERS["google-antigravity"].providerConfig.liveModels).toBe(true); expect(OAUTH_PROVIDERS["google-antigravity"].providerConfig.defaultModel).toBe("gemini-3.6-flash"); // Collapsed picker: base models only, no effort-suffix variants. expect(OAUTH_PROVIDERS["google-antigravity"].providerConfig.models).toContain("gemini-3.6-flash"); From 8f26e986d558e32a6bf7ca46d78d34f295f94ca5 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 9 Aug 2026 02:46:12 +0900 Subject: [PATCH 27/77] docs(devlog): record WP15, WP16, WP3, WP5 and lane D of the bug campaign (#1307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five units, and the honest summary is that adversarial review changed my answer far more often than it polished it. 023 WP15 — three contributor fixes republished on dev with Co-authored-by preserved. Records that I read #1244's CI as in_progress and built a "watch" disposition on a run that had already concluded failure. 024 WP16 — #1273 is two defects. I designed the second fix three times and review caught the first two losing user data: whole-array reconciliation resurrects a deleted provider's rows, and keying on routedSlug duplicates renamed ones. Shipped defect 1, left defect 2 open with a diagnosis rather than landing a third attempt in a config-persistence path. 025 WP3 — #1185 was red and right; its crash was a Bun EEXIST in a file its one-file diff cannot reach. Also records a workflow snippet I suggested that interpolated an expression straight into shell, while reviewing a security-class change. 026 WP5 — #1244's author answered my defect report with neither fix I proposed, and was right. My stated reason for agreeing was wrong too: tsconfig.json has include:[src], so the typecheck I cited as caller-sweep evidence never read tests/ at all. 027 lane D — closes the catalog sequence, and records six PRs merged to dev without the approval MAINTAINERS.md requires. I logged every workflow-run approval meticulously against its head SHA, which is what made the missing pull-request approval feel handled. Filed as #1306 rather than back-filled. --- .../023_wp15_1244_and_green_five.md | 574 ++++++++++++++++ .../024_wp16_ghost_custom_models.md | 624 ++++++++++++++++++ .../025_wp3_lane_c_ci_workflows.md | 164 +++++ .../260808_bug_campaign/026_wp5_large_solo.md | 94 +++ .../027_wp4_lane_d_close.md | 152 +++++ 5 files changed, 1608 insertions(+) create mode 100644 devlog/_plan/260808_bug_campaign/023_wp15_1244_and_green_five.md create mode 100644 devlog/_plan/260808_bug_campaign/024_wp16_ghost_custom_models.md create mode 100644 devlog/_plan/260808_bug_campaign/025_wp3_lane_c_ci_workflows.md create mode 100644 devlog/_plan/260808_bug_campaign/026_wp5_large_solo.md create mode 100644 devlog/_plan/260808_bug_campaign/027_wp4_lane_d_close.md diff --git a/devlog/_plan/260808_bug_campaign/023_wp15_1244_and_green_five.md b/devlog/_plan/260808_bug_campaign/023_wp15_1244_and_green_five.md new file mode 100644 index 000000000..1dc8953c8 --- /dev/null +++ b/devlog/_plan/260808_bug_campaign/023_wp15_1244_and_green_five.md @@ -0,0 +1,574 @@ +# WP15 — #1244 rebases itself, and five green contributor fixes get republished + +## What changed since WP14 + +WP15 opened as "resolve #1244's 22-hunk conflict by hand". That work no longer +exists. The PR head moved from `b413f8bff` to `15545b3d1` and the author +collapsed the branch onto the current `dev` tip: + +``` +$ git fetch origin pull/1244/head && git merge-base --is-ancestor origin/dev FETCH_HEAD && echo on-dev +on-dev +$ git log --oneline FETCH_HEAD -1 +15545b3d1 fix(codex): preserve routed models in desktop picker +``` + +`mergeStateStatus` is `UNSTABLE` rather than `DIRTY`, `isDraft` is now `false`, +and the 57-file diff no longer defines `mergeCatalogModelsWithNativeRecovery` +locally. The WP14 hypothesis — that resolving 22 hunks across +`src/codex/catalog/sync.ts` and `src/codex/convergence.ts` requires re-deciding +the author's design against the merged #1212 convergence work — is moot. The +author did that re-decision themselves. + +Consequence for the campaign: #1244 needs no maintainer rebase. It needs CI to +finish (`Cross-platform CI` and `Service lifecycle` were both `in_progress`) +and then a normal merge decision. That is a watch, not a work item. + +## The actual WP15 unit + +Five contributor PRs sit one or two commits ahead of a `dev` they are far +behind, and all five still sit in draft because the four-box readiness +checklist is the contributor's own attestation, which I will not tick for them +(`.github/workflows/enforce-pr-target.yml:516`). + +| PR | Author | Head | ahead/behind dev | Cross-platform CI at head | +|----|--------|------|------------------|---------------------------| +| #1189 | luvs01 | `d5242a231` | 2 / 300 | success | +| #1195 | luvs01 | `6eff3f6a5` | 2 / 300 | success | +| #1169 | TyroneXie | `d8968b7e6` | 1 / 335 | success | +| #1187 | luvs01 | `36cffcef6` | 2 / 9 | action_required | +| #1184 | luvs01 | `a2eda3b94` | 1 / 16 | action_required | + +Read that column through the WP1 rule: `gh pr checks` hides `action_required`, +so the source of truth is +`gh api "repos/lidge-jun/opencodex/actions/runs?head_sha=$sha"`, and `status` is +not `conclusion`. #1187 and #1184 are unapproved, not failing. + +All five apply cleanly onto `3ad5bb6bd`: + +``` +PR#1189 APPLIES CLEAN +PR#1187 APPLIES CLEAN +PR#1184 APPLIES CLEAN +PR#1195 APPLIES CLEAN +PR#1169 APPLIES CLEAN +``` + +"Applies clean" is textual, not semantic. The three deep-behind branches +(#1189, #1195, #1169 — 300+ commits) are exactly the case where a clean apply +can still be wrong, because `dev` may have moved the surrounding contract +without touching the same lines. Each gets a contract check below, and a clean +apply alone is not accepted as evidence for any of them. + +## Diff-level plan + +Republish protocol is `003_republish_protocol.md` unchanged: fresh worktree from +`origin/dev`, apply the author's net diff, one commit authored by the maintainer +with a trailer preserving the contributor, a PR body naming the source PR and +mentioning the author, then merge on green. + +### WP15-A — #1187 and #1184: approve at head, merge in place + +> **Corrected by audit B2 — see the audit-fold section below. The heading is +> wrong: these cannot be merged in place. Read WP15-A′.** + +Nine and sixteen commits behind, both under the gate's 10-commit tolerance after +a rebase, both tiny. No republish is warranted; the correct action is to unblock +CI. + +1. Re-read each PR head immediately before acting; log `run.head_sha` and + `pr.headRefOid` as separate columns in `.tmp/ocx_approval_ledger.tsv` with + `MATCH`/`SKIP`. A `SKIP` means the author pushed inside the window and the + approval would target a stale commit — abort that row. +2. Approve the `action_required` Cross-platform CI run for a `MATCH` row. +3. Await `conclusion == "success"`. A `failure` gets diagnosed, never guessed. + +Acceptance: both PRs have a `success` Cross-platform CI at the exact head the PR +points at, recorded with both SHAs. + +What this does not do: it does not make them mergeable by policy, because the +contributor checklist stays theirs. Approval only removes the gate that stops +them from proving box 1. + +### WP15-B — #1189: republish as `codex/260808-1189-history-stream-ingest` + +Net diff: `src/routing/history/indexer.ts`, +`tests/request-history-index.test.ts`. + +It replaces `readCompleteTail` — which allocated `size - indexedOffset` in one +shot — with a 64 KiB chunked reader that assembles records across chunk +boundaries and omits complete records above a 1 MiB projection bound +(`REQUEST_HISTORY_READ_CHUNK_BYTES`, `REQUEST_HISTORY_MAX_RECORD_BYTES`). It +deletes `ingestText` and folds line handling into `ingestSourceTail`. + +Contract checks, required because the branch is 300 behind: + +- `rg 'ingestText|readCompleteTail' src tests` on `dev` must show no caller + outside `indexer.ts`; a surviving caller means the deletion breaks it. +- `insert.finalize()` must still run unconditionally in `finally`. The Windows + file-lock note in the deleted code is load-bearing and CI runs Windows. +- Offset accounting: `nextOffset` may only advance past a `\n`, so a torn final + record is re-read rather than skipped. + +Verification: `bun test tests/request-history-index.test.ts`, +`bun run typecheck`, and an ablation reverting the 1 MiB bound that shows the +oversized-record test failing. A passing suite without the ablation is not +accepted. + +### WP15-C — #1195: republish as `codex/260808-1195-unbound-quota-unknown` + +Net diff: `src/router.ts`, +`src/server/management/routing-profile-routes.ts`, +`tests/quota-scoring.test.ts`, English routing docs. + +It deletes the same ~20-line block from both files: a policy candidate no longer +takes `codexAccountId`/`codexAccountPlan` from +`getEffectiveActiveCodexAccountId` when the Codex provider is in pool mode, and +no longer takes `accountRef` from `getAccountSet("anthropic")`. The rationale is +a real ordering defect — policy evaluation runs before Pool/Direct identity, +thread affinity, and Anthropic session affinity resolve, so a candidate can be +scored with account A's quota and executed on account B. + +Contract checks: + +- The two blocks must still be identical on `dev`. If `dev` already changed + either one, the delete is no longer symmetric and the PR is stale. +- `rg 'getEffectiveActiveCodexAccountId|getAccountSet'` must come back empty for + both files, and the now-unused imports must be gone or `typecheck` fails. +- Live-vs-dry-run parity: both paths must emit the same evidence shape, which is + the property `tests/quota-scoring.test.ts` is extended to hold. + +Verification: quota, policy-execution, routing-profile, and explainability +suites; `bun run typecheck`; an ablation restoring one block only, to show the +parity test fails asymmetrically. + +### WP15-D — #1169: republish as `codex/260808-1169-shim-routing-warning` + +Net diff: `src/cli/codex-shim-readiness.ts` (new), `src/cli/index.ts`, +`tests/codex-shim-readiness.test.ts`, English and zh-CN lifecycle docs. + +`ocx codex-shim install` reports clean success even when it cannot prove Codex +routes through OpenCodex. The change downgrades that to a warning for an +external `model_provider`, a user-owned local or remote gateway, or +unverifiable routing, and warns when proxy variables exist only in the current +process while `config.proxy` is unset. Advisory only: same exit code, and the +shim still fail-open execs the real launcher. + +Highest risk of the three at 335 commits behind, and it touches +`src/cli/index.ts`, which this campaign already modified. Contract checks: + +- The `codex-shim install` call site in `src/cli/index.ts` must still have the + shape the patch expects; confirm by reading the applied hunk rather than + trusting the apply. +- Privacy is the blocking property: no proxy URL, token, or account identifier + may reach stdout. `bun run privacy:scan` plus the test's own assertion inside + isolated `CODEX_HOME`/`OPENCODEX_HOME` directories. +- The exit code must be unchanged on the warning path. Assert it, because + "advisory only" is the whole safety argument. + +Verification: `bun test tests/codex-shim-readiness.test.ts`, +`bun run typecheck`, `bun run privacy:scan`, and a real CLI install in a temp +home showing the warning text with no secret in it. + +### WP15-E — #1244 watch + +> **Corrected by audit B1. #1244's Cross-platform CI at `15545b3d1` is +> `failure`, not `in_progress`. Read WP15-E′.** + +No code work. Poll the two `in_progress` runs at `15545b3d1`; merge on +`success`, diagnose and comment on `failure`. If the head moves again, re-read +it before acting. Record the outcome either way. + +## Acceptance criteria + +1. #1187 and #1184 have `success` Cross-platform CI at their exact current head, + both SHAs logged and matched in `.tmp/ocx_approval_ledger.tsv`. +2. Three new PRs exist for #1189, #1195, #1169, each with a `Co-authored-by` + trailer naming the original author, each mentioning them, each filling all + three PR-template sections. +3. Each republish carries a fresh focused-test result and an ablation that fails + without the fix. +4. `bun run typecheck` clean on each republished branch; `privacy:scan` clean on + #1169's. +5. #1244's CI outcome at `15545b3d1` is recorded with a disposition. +6. No contributor readiness checkbox is ticked by me anywhere. + +## Faults to avoid, restated because I have committed each one + +- Merging without a real green (#1202, WP1). +- Selecting by branch name instead of head SHA; the ledger caught this once. +- Ticking a contributor's readiness box. Done once, reverted. +- Claiming a root cause from plausible commit messages without + `git merge-base --is-ancestor` (#1178). +- Trusting `gh pr checks` to surface `action_required`. It does not. +- **Reading a run's `status` and stopping there.** I recorded #1244 as + `in_progress` and built a "watch" around it. By the time the plan was + audited the run had concluded `failure`, so the plan shipped a wrong + disposition for the single largest PR in it. Re-read `conclusion` at the + moment of the decision, not at the moment of the survey. + +--- + +# Audit fold — six blockers, all accepted + +A `gpt-5.6-terra` reviewer audited the plan above against live GitHub state and +returned `VERDICT: fail` with B1–B6. Every one is accepted without rebuttal. +The corrections below supersede the corresponding sections. + +## B1 — #1244 is failing CI, and the failure is in the #1212 seam + +This is the blocker that matters. The survey above recorded two runs as +`in_progress`; the reviewer read the conclusion: + +``` +$ gh api "repos/lidge-jun/opencodex/actions/runs?head_sha=15545b3d1..." \ + -q '.workflow_runs[]|[.id,.name,.status,.conclusion]|@tsv' +31256061011 Issue quality tests completed success +31256061013 Service lifecycle completed success +31256062356 Enforce PR target branch completed success +31256062366 PR Labeler completed success +31256063542 React Doctor completed success +31256063557 Cross-platform CI completed failure +``` + +The failing shard throws `TypeError: suppressedBareNativeSlugs.has` at +`src/codex/catalog/sync.ts:427`, driven by `tests/codex-v2-gate.test.ts:1211`. +The new required input is destructured at `sync.ts:385` **without a default**, +so any caller on `dev` that does not pass it gets `undefined` and dies on +`.has`. + +That is precisely the semantic conflict WP14 suspected and this plan dismissed. +The author's textual rebase merged cleanly *and* broke the contract, which is +the exact failure mode the plan claimed to guard against for the three +republishes while waiving it for #1244. A clean `merge-base --is-ancestor` said +nothing about whether every observed-state caller was updated. + +### WP15-E′ — #1244: report the failure, do not merge + +1. Merge is prohibited at `15545b3d1` and at any later head until a + Cross-platform CI run at that exact head concludes `success`. +2. Comment on #1244 with the run id, the file:line, the failing test, and the + missing-default diagnosis. State the fix shape (either default the + destructured input or update every caller) without asserting which one the + author should pick — the callers are their design. +3. Re-audit the #1212-adjacent observed-state callers + (`buildCatalogEntriesFromObservedState`, + `mergeCatalogEntriesFromObservedState`, `shouldUpgradeToUpstreamEntry`) + before any future merge decision, because a single missing default proves the + caller sweep was incomplete. +4. Disposition: **awaiting author**, with a concrete defect. Not a watch. + +## B2 — approval is not merge-readiness + +`.github/workflows/enforce-pr-target.yml:820` sets `mustDraft` while a +contributor checklist is incomplete, and `:1027`–`:1042` preserve draft status +until every box is ticked. So approving CI does **not** make #1187 or #1184 +mergeable, and WP15-A's heading claimed an outcome the gate forbids. + +### WP15-A′ — approve CI only; await contributor readiness + +Steps 1–3 of WP15-A stand unchanged (SHA-matched ledger, approve `MATCH` rows, +await `conclusion`). What changes is the claim: + +- Acceptance is narrowed to: a `success` Cross-platform CI exists at the exact + current head, both SHAs logged and matched. +- Terminal disposition is **awaiting author**, not merged. The four boxes are + the contributor's attestation and stay theirs. +- The PR comment must say what approval did and did not do, so the author is not + left thinking the maintainer unblocked a merge. + +## B3 — focused tests are below the mandatory gate + +`003_republish_protocol.md:286` requires the full suite when a change touches +routing, adapters, config, or the server; `AGENTS.md:228` requires it +independently. All three republishes touch routing or the CLI, so the +focused-test acceptance at `:110`, `:138`, and `:166` was under-specified. + +**Correction:** `bun run test` (full suite) is required on each republished +branch before its PR is opened, and again after any base movement that forces a +re-apply. The focused test and the ablation stay — they are additional +evidence, not a substitute. + +## B4 — the disposable-worktree boundary was implied, not enforced + +The plan says "fresh worktree" at `:65`, but the protocol it delegates to runs +`git switch -c`, `cherry-pick`, and `commit` in whatever checkout executes it +(`003_republish_protocol.md:39`–`:48`). Run literally in this checkout, that +would touch the user's dirty files. + +**Correction, binding for every remaining work-phase:** + +- Every republish runs in `git worktree add --detach "$(mktemp -d)/"`. +- `git switch`, `git commit`, and any write inside + `/Users/jun/.codex/worktrees/1a75/opencodex` are prohibited except for + `devlog/` documents. +- `scripts/generate-jawcode-metadata.ts`, + `src/generated/jawcode-model-metadata.ts`, + `tests/jawcode-metadata-sync.test.ts`, and `scripts/jawcode-models.json` are + the user's uncommitted work and are never staged, stashed, or reverted. +- Test and ledger artifacts live inside the disposable worktree; it is removed + when the phase closes. + +## B5 — #1195's parity claim is not covered by its own test + +The PR only changes execution-path assertions in `tests/quota-scoring.test.ts` +(runtime assertion at `:198`–`:209`). The management dry-run test at +`tests/routing-profile.test.ts:451`–`:477` covers a candidate with an +explicitly supplied `codexAccountId` — never an *unbound* candidate while a +pool account is active. That is the whole defect, and no test would catch its +return. + +**Correction:** the republished #1195 must add paired regressions — runtime and +management dry-run — asserting that an unbound Codex candidate and an unbound +Anthropic candidate both keep `quota.known === false` while a global active +account exists, and that explicit account-qualified evidence stays known. The +ablation restores one of the two deleted blocks and must fail the new pair. +Without that pair the PR body may not claim parity. + +## B6 — the inventory is stale; two bugs opened after the sweep + +The campaign's own objective is a terminal disposition for *every* open bug +issue and PR. Two were opened after the inventory and appear nowhere in it: + +| # | State | Labels | Note | +|---|-------|--------|------| +| #1273 | OPEN | bug | ghost custom models survive provider removal; full-config PUT resurrects deleted `customModels` | +| #1278 | OPEN | bug, platform, install | Windows: transient PowerShell console window on identity lookup; distinct from #1236 | +| #1279 | OPEN | — | non-draft fix PR for #1278 (`43b6b824c`, wade19990814-hue) | +| #1283 | OPEN | bug, gui | grok's limit is weekly not monthly (opened 12:06Z, *after* the re-audit's own B6 list) | + +**Correction:** a live inventory resweep runs immediately before execution, and +#1273, #1278, #1279, and #1283 are added to the disposition matrix. WP15 does not +close them silently; if they cannot be dispositioned inside this phase they +become the next work-phase with that stated explicitly. + +### B6, second round — a fixed list cannot satisfy a moving inventory + +The re-audit reopened B6 after I had already folded it. Between the first audit +and the second, #1283 opened. My correction had enumerated three numbers, so it +would have passed its own criterion while dropping a bug that existed before the +phase closed. Enumerating is the wrong shape for this criterion. + +**Correction of the correction:** criterion 6 is no longer a list. It requires a +*recorded live resweep, executed last*, whose output is pasted into the closing +document, covering every then-open `bug`-labeled issue and every associated fix +PR. A number opened after that resweep is out of scope by timestamp, and the +resweep output is what proves the boundary rather than my own recollection. + +### B6, third round — an approximate timestamp and a missing PR side + +The re-audit rejected my resweep too, on two counts, both fair. I had written the +boundary as `2026-08-08T12:0xZ` — an approximation is not a boundary, and it +cannot decide whether a given issue was in scope. And I had run only +`gh issue list`, so the PR half of "every open bug issue *and* its fix PRs" was +unproven. + +**Resweep, exact boundary and deterministic issue → open-PR mapping.** +`RESWEEP_AT=2026-08-08T12:21:41Z`, derived per issue from the cross-referenced +timeline events, open pull requests only: + +| Issue | Open fix PR(s) | Disposition | +|-------|----------------|-------------| +| #1283 | none | → WP16 (new, `bug`+`gui`, opened 12:06:03Z) | +| #1278 | #1279 | → WP16 (non-draft PR already open) | +| #1273 | none | → WP16 (new) | +| #1236 | #1268, #1279 | tracking; #1278 is explicitly distinct from it | +| #1230 | #1269 | stays open — the `handleEnsure` gap at `src/cli/index.ts:441` is unfixed | +| #1229 | none | tracking | +| #1222 | none | tracking | +| #1213 | none | tracking | +| #1196 | #1270 | awaiting contributor (two blockers commented) | +| #1193 | #1205 | rerun needed — the run ended `cancelled`, not `failure` | +| #1190 | #1210 | CI green, awaiting contributor checklist | +| #1162 | none | tracking | +| #1145 | none | tracking | +| #1128 | none | tracking, reporter capture requested | +| #1059 | #1272 | CI green, awaiting contributor checklist | +| #1024 | none | tracking, reporter capture requested | +| #904, #796, #418, #417, #241, #92 | none | long-lived tracking | + +That is 22 open `bug` issues, each with either a disposition already recorded in +this unit or an explicit hand-off. + +One row needs its cross-reference stated precisely rather than assumed. #241's +timeline lists only closed PRs (#298, #999, #1056, #1147, #1150), so the +mechanical mapping correctly reports no open fix PR. #1244 does not link #241. + +The chain has two links of different strength, and my first attempt at this +paragraph flattened both into "exists only in prose", which the audit +corrected: **#241 → #1056 is a real timeline cross-reference; #1056 → #1244 is +inferred solely from #1244's `Supersedes #1056` body text.** Only the second +hop is prose. I had also asserted a direct #1244 → #241 link earlier in this +campaign without reading the timeline; that was wrong, and this is the +corrected form. + +The scope boundary is the timestamp above. Anything opened after +`12:21:41Z` is out of this phase by construction, and that is provable from the +recorded value rather than from my recollection. + +## Revised acceptance criteria for WP15 + +1. #1187 and #1184: `success` Cross-platform CI at the exact current head, both + SHAs logged and matched. Disposition recorded as **awaiting author**, with no + merge claim and no box ticked by me. +2. Three new PRs for #1189, #1195, #1169, each with a `Co-authored-by` trailer + naming the original author, each mentioning them, each filling all three + template sections. +3. Each republish: full `bun run test` green, plus a focused test, plus an + ablation that fails without the fix. #1195 additionally carries the paired + parity regressions from B5. +4. `bun run typecheck` clean on each branch; `privacy:scan` clean on #1169's. +5. #1244: the CI `failure` at `15545b3d1` is reported on the PR with run id, + file:line, and the missing-default diagnosis. Merge prohibited. +6. #1273, #1278, #1279 appear in the disposition matrix with either a terminal + disposition or an explicit hand-off to the next work-phase. + *(Superseded by B6 round two: a recorded live resweep, run last, must be + pasted in, and every then-open bug issue must have a disposition or a named + hand-off. No fixed list.)* +7. All code work happened in `mktemp -d` worktrees; the user's four dirty files + are untouched (`git status --short` proves it). + +--- + +# Execution record + +## The audit moved the base out from under the work + +Round four caught something none of the earlier rounds could: while I was +folding blockers, `origin/dev` moved from `3ad5bb6bd` to `f5147cbc8`. Every +test result on this page — three full suites, two ablations, three typechecks — +was measured against a base that no longer existed. The reviewer's instruction +was to treat all of it as stale and redo it after rebasing, which is correct and +which I did. + +The cost of skipping that step would have been three PRs whose "Verification" +sections cited numbers from a base the reviewer could not reproduce. That is the +same class of fault as merging #1202 without a real green, just better hidden. + +Re-verified on `f5147cbc8`: + +| Branch | Full suite | Focused | Extra | +|--------|-----------|---------|-------| +| `codex/260808-1189-history-stream-ingest` | 9991 pass / 7 skip / 0 fail, 625 files | 20/20 | ablation 19/1 then restored 20/0 | +| `codex/260808-1195-unbound-quota-unknown` | 9992 pass / 7 skip / 0 fail, 625 files | 31/31 | ablation 27/4 at identical scope | +| `codex/260808-1169-shim-routing-warning` | 9994 pass / 7 skip / 0 fail, 626 files | 5/5 | `privacy:scan` passed | + +`bun run typecheck` clean on all three. The prepush hook then ran the full suite +a second time per branch and passed each one, which is why the pushes took +roughly six minutes apiece. + +## Published + +| New PR | Republishes | Author | Head | +|--------|-------------|--------|------| +| #1287 | #1189 | luvs01 | `02ec799fe` | +| #1288 | #1195 | luvs01 | `3fc962f2c` | +| #1289 | #1169 | TyroneXie | `eac814346` | + +All three opened non-draft against `dev`, `MERGEABLE`, and `Enforce PR target +branch` green on each. Trailer evidence: + +``` +$ git log --format='%h %s%n %(trailers:key=Co-authored-by,valueonly)' origin/dev..HEAD +02ec799fe fix(history): stream request-history index ingestion (#1189) + luvs01 <27862058+luvs01@users.noreply.github.com> +3fc962f2c test(routing): prove the management dry-run leaves unbound candidates unknown + +0c745be36 fix(routing): keep unbound account quota unknown (#1195) + luvs01 <27862058+luvs01@users.noreply.github.com> +eac814346 fix(codex): warn when codex-shim install cannot prove routing (#1169) + TyroneXie <328347833@qq.com> +``` + +The blank trailer line on `3fc962f2c` is deliberate and is the point of B5's +attribution requirement: those two dry-run tests are mine, not luvs01's, so they +are a separate commit with no co-author trailer and an explicit paragraph in +#1288's body saying so. Folding them into the contributor's commit would have +attributed my code to them; leaving them out would have shipped an unproven +parity claim. + +## Actions taken on existing PRs + +- **#1187, #1184** — approved the `action_required` Cross-platform CI at + SHA-matched heads (`36cffcef6`, `a2eda3b94`; both `MATCH` in + `.tmp/ocx_approval_ledger.tsv`). Commented on both that approval unblocks CI + and nothing else, and that the four boxes stay theirs. Disposition: **awaiting + author**. +- **#1244** — commented with run `31256063557`, the `TypeError` at + `src/codex/catalog/sync.ts:427`, the triggering test at + `tests/codex-v2-gate.test.ts:1211`, and the missing default at `sync.ts:385`. + Named both fix shapes without choosing for them, and flagged that the same + caller-sweep gap may exist for the other inputs added in that commit. Merge + held. Disposition: **awaiting author, with a concrete defect**. +- **#1189, #1195, #1169** — commented on each that it was republished, by which + PR, with what verification, and that the author may take it back if they + prefer to drive it themselves. + +## What WP15 did not do + +#1283, #1278/#1279, and #1273 are dispositioned as hand-offs to WP16, not as +closed. Naming them here is the honest form of that; the resweep table above is +what makes the boundary checkable rather than asserted. + +## Merged + +All three landed on `dev`, and the `Co-authored-by` trailer survived each +squash — which is the property that matters, because the squash is where +contributor credit usually gets lost: + +``` +57ea8df47 fix(routing): keep unbound account quota unknown (#1195) (#1288) | luvs01 +5aa197112 fix(codex): warn when codex-shim install cannot prove routing (#1169) (#1289) | TyroneXie +2cb8eddd4 fix(history): stream request-history index ingestion (#1189) (#1287) | luvs01 +``` + +#1189, #1195, and #1169 were closed as superseded, each with a comment naming +the landed SHA and confirming the credit. #1195's closing comment states +separately that the maintainer test commit is mine and their fix commit is +unmodified. + +### #1288 needed two reruns, and the reason is worth recording + +Cross-platform CI at `3fc962f2c` came back `cancelled` twice. A `cancelled` is +not a `failure` — the four-state rule says rerun — but twice in a row is a +signal rather than noise, so I read the job log instead of firing a third +rerun blind. `test 3/4` hung at `tests/cli-restart-health.test.ts` and was +killed by the runner after ~14 minutes. + +The check that made this safe was comparing against `dev` itself: + +``` +31259885820 dev all-shards-ok +31259450263 dev cancelled test 3/4=cancelled +31259447622 dev cancelled test 1..4/4=cancelled +31256617398 dev success all-shards-ok +``` + +The same shard cancels on `dev` with no PR involved, so it is runner flake, not +something #1288 introduced. `rerun-failed-jobs` then returned all four shards +green. Had I not checked `dev`, "rerun until green" would have been +indistinguishable from hiding a real defect — which is exactly the failure mode +the four-state rule exists to prevent. + +## Final resweep — `FINAL_RESWEEP_AT=2026-08-08T13:48:43Z` + +Run last, as criterion 6 requires. Twenty-two open `bug` issues, unchanged in +membership from the 12:21:41Z sweep, so nothing opened during execution. +#1283, #1278/#1279, and #1273 remain the undispositioned three and pass to +WP16. + +## Acceptance, checked + +1. #1187, #1184 — Cross-platform CI `success` at `36cffcef6` and `a2eda3b94`, + both `MATCH` in the ledger. Awaiting author. **Met.** +2. Three new PRs with trailers and mentions, all template sections filled. + **Met.** +3. Full suite + focused test + ablation on each; #1195 carries the B5 parity + pair. **Met.** +4. `typecheck` clean on all three; `privacy:scan` clean on #1289's. **Met.** +5. #1244's `failure` reported with run id, file:line, and the missing-default + diagnosis; merge held. **Met.** +6. Final resweep recorded above with a disposition or hand-off per row. + **Met.** +7. All code work in `mktemp -d` worktrees; `git status --short` still shows + exactly the user's four untouched files. **Met.** diff --git a/devlog/_plan/260808_bug_campaign/024_wp16_ghost_custom_models.md b/devlog/_plan/260808_bug_campaign/024_wp16_ghost_custom_models.md new file mode 100644 index 000000000..ab9083448 --- /dev/null +++ b/devlog/_plan/260808_bug_campaign/024_wp16_ghost_custom_models.md @@ -0,0 +1,624 @@ +# WP16 — ghost custom models (#1273), and closing out the campaign's tail + +## Scope + +Three items survived WP15 as hand-offs. One is already done, one is a +disposition, and one is a real two-part defect that needs a patch. + +| Item | State entering WP16 | Outcome | +|------|--------------------|---------| +| #1283 grok weekly limit | OPEN, but fixed on `dev` by #1290 | closed with the landed SHA | +| #1278 / #1279 Windows console flash | PR open, CI unapproved | CI approved at SHA-matched head | +| #1273 ghost custom models | OPEN, no fix PR | **the work of this phase** | + +### #1283 — already fixed, closed manually + +`5222f354a fix(quota): prefer Grok weekly credits for xAI dashboard (#1290)` +landed on `dev` at 13:35Z and does exactly what the report asked: prefer +`GET /v1/billing?format=credits` and map SuperGrok's weekly window to +`weeklyPercent`/`weeklyResetAt`, with the legacy 30-day endpoint demoted to a +fallback (`src/providers/quota.ts:49`, `:592`). + +The PR body said `Closes #1283`, but PRs here target `dev` and GitHub only +auto-closes on merges into the default branch, so the issue sat open with its +fix already shipped. Closed by hand with the SHA. This is a recurring trap in +this repository and it is why `AGENTS.md` tells contributors to close linked +issues manually. + +One loose thread worth recording: #1290's own description noted a pre-existing +`tests/translator-budget.test.ts` failure on the tip and pushed with +`--no-verify`. Checked on a clean worktree at `dev`: **13 pass / 0 fail**. The +failure was local to that environment, not on `dev`, so nothing to chase. + +### #1278 / #1279 — approved, awaiting author + +#1279 (`fix(windows): eliminate console windows from proxy-internal identity & +process lookups`, wade19990814-hue, head `43b6b824c`) is non-draft and +`MERGEABLE`, touching seven Windows source files and four test files. Its +Cross-platform CI sat at `action_required`, invisible to `gh pr checks`. + +Approved at a SHA-matched head (`43b6b824c` == run head, `MATCH` in +`.tmp/ocx_approval_ledger.tsv`). Disposition is **awaiting CI, then review** — +the change is Windows-specific and this campaign has no Windows host, so CI is +the only evidence available and the review will have to lean on it. + +## #1273 — the actual defect + +The report describes two defects. Both reproduce in the source; neither is a +false positive. + +### Defect 1 — provider removal orphans `config.customModels` + +Both removal paths delete only the provider record: + +- `src/cli/provider.ts:304` — `delete config.providers[name]; validateAndSave(config);` +- `src/server/management/provider-routes.ts:617` — `delete config.providers[name];` + followed by `setProviderContextCap`, `save`, `reconcileLiveStateStores`, + `clearModelCache(name)`, `convergeCodexCatalog()`. + +Note what the management path *does* clean up: context caps, the model cache, +and the catalog. It walks right past `config.customModels`. So a custom model +for the removed provider stays in the config, keeps appearing in `/api/models`, +and keeps being emitted into the Codex catalog — a row pointing at a provider +that no longer exists. + +### Defect 2 — a stale in-memory config wins a whole-document write + +`saveConfigPreservingClaudeCode` (`src/config.ts:2710`) takes one authoritative +pre-write read of the on-disk config and uses it for exactly two reconciliations: + +- `claudeCode` (`:2716`–`:2726`): if disk changed and we did not, adopt disk. +- the live server binding (`:2731`–`:2739`): port/hostname come from disk. + +`customModels` gets neither. `projectCustomModelCatalogMigration` +(`src/codex/custom-model-catalog-migration.ts`) *does* consult the persisted +config, but only to project the `customModelCatalogMigration` ownership marker — +it reads `customModels` to classify legacy slugs and never writes the array +back. The candidate's array passes through untouched into +`persistConfigUnlocked`. + +So the reporter's step 4 is exactly right: a `PUT /api/shadow-call-settings` +from a process whose config predates a CLI deletion re-persists the whole stale +document, and the deleted rows return. Their diff even shows the cooperating +save path working correctly (generation bumped, catalog mtime matching) — the +write was well-formed, it just wrote the wrong document. + +**The asymmetry is the bug.** Two fields already get last-writer-wins protection +because they are known to be mutated by other processes. `customModels` is +mutated by `ocx models remove` from a different process and got no such +treatment. + +## Diff-level plan + +### WP16-A — `src/config.ts`: reconcile `customModels` against disk + +Add a third reconciliation next to the `claudeCode` block, using the same +already-taken `onDisk` read (`:2715`) — no second read, since the comment there +correctly warns that a second read could observe different bytes. + +Shape, mirroring the `claudeCode` baseline logic: + +- Track a `customModelsBaseline` per config object, set when the config is + loaded, exactly as `claudeCodeBaseline` is. +- On write: if the on-disk array differs from the baseline **and** the + in-memory array equals the baseline, adopt the on-disk array. That is + "someone else changed it and we did not", which is precisely the reporter's + scenario. +- If both changed, the in-memory value wins and we do not silently merge. A + merge would invent an intent neither writer expressed; last-writer-wins on a + genuine concurrent edit is the same rule `claudeCode` already uses. + +Acceptance: a test that loads a config, deletes a custom model on disk out of +band, then performs an unrelated `PUT`-shaped save from the stale object, and +asserts the deleted row does not return. + +### WP16-B — both removal paths drop the provider's custom models + +`src/cli/provider.ts:304` and `src/server/management/provider-routes.ts:617` +each filter `config.customModels` by `provider !== name` before saving. The +management path already clears the model cache and reconverges the catalog, so +the ghost disappears from `/api/models` and the catalog in the same write. + +**Open question, answered before writing the patch.** Does anything rely on a +`customModels` row outliving its provider — re-adding the provider and +expecting its models back, for instance? If so, deleting would be wrong and the +fix would belong at read time. + +The answer is in provider *rename*. `rewriteProviderReferences` +(`src/providers/provider-id-rewrite.ts:34`) explicitly rewrites +`customModels[].provider` alongside combo targets and Claude tier maps +(`:97`–`:101`). So this array is already designed to track the provider +lifecycle; rename follows it and remove simply does not. No test anywhere +expects rows to survive removal and be restored on re-add. + +That makes deletion the consistent fix rather than a judgement call, and it +reframes defect 1: not a missing feature, but a lifecycle hook that one of two +sibling operations forgot. Consumers confirm the same shape — +`src/codex/catalog/provider-fetch.ts` emits every row into the catalog keyed by +`routedSlug`, and `src/server/management/model-rows.ts` lists every row in the +dashboard, neither checking that the provider still exists. + +### WP16-C — verification + +- Focused: new tests in the config and provider-removal suites. +- Ablation: revert each half independently; the matching test must fail and the + other must not. Two defects, two independent proofs. +- Full `bun run test`, `bun run typecheck`, `bun run privacy:scan`. +- Republish protocol: `mktemp -d` worktree, no writes in the dirty checkout + outside `devlog/`. + +This is a maintainer-authored fix with no contributor PR to preserve, so there +is no `Co-authored-by` trailer. The PR body credits the reporter for a +reproduction that included a before/after config diff and the evidence that the +cooperating save path itself was healthy — that is what made the second defect +findable rather than a vague "settings sometimes revert". + +## Acceptance criteria + +1. #1283 closed with the landed SHA and an explanation of the manual close. +2. #1279's CI approved at a SHA-matched head, logged, disposition recorded. +3. Both #1273 defects fixed, each with its own regression test and its own + ablation. +4. Full suite, typecheck, and privacy scan green on the rebased branch. +5. PR opened against `dev` with all three template sections and the reporter + credited. +6. The consumer question in WP16-B answered with file:line evidence before the + patch is written. + +--- + +# Audit fold — five blockers, all accepted + +A `gpt-5.6-terra` reviewer returned `VERDICT: fail` on the plan above with +B1–B5. Every one is accepted. B3 is the important one: it finds a case where my +proposed rule loses user data, which is worse than the bug it was written to +fix. + +## B3 — "memory wins if both changed" resurrects a deleted provider's model + +My rule was whole-array last-writer-wins: adopt disk when disk changed and +memory did not, otherwise keep memory. The reviewer supplied the case that +breaks it. + +Disk deletes provider **P** (and, after WP16-B, P's custom models). Meanwhile +the in-memory process independently edits an unrelated custom model **Q** — a +legitimate edit through `/api/models`. Now *both* arrays differ from the +baseline, so my rule keeps memory wholesale, and P's custom model comes back. +The exact ghost row #1273 is about, reintroduced by the fix for #1273. + +A whole-array comparison cannot distinguish "I edited Q" from "I am asserting +the entire array including P". The array is a keyed collection and has to be +reconciled as one: + +- Reconcile per row, keyed by `routedSlug(provider, modelId)`, three-way + against the baseline: a row deleted on disk and untouched in memory stays + deleted; a row edited in memory is kept; a row added on either side is kept. +- Then prune any surviving row whose provider is absent from the config being + written. That prune is the lifecycle invariant from WP16-B applied at the + write boundary, so it holds no matter which path produced the array. + +Required regression: disk deletes P while memory edits Q, then an unrelated +save. Q keeps its edit, P's row does not return. + +## B1 — the same staleness applies to `providers`, not just `customModels` + +I scoped the fix to `customModels` because that is what the issue reported. But +`saveConfigPreservingClaudeCode` writes the *whole document*, so the identical +stale-write path resurrects `config.providers[P]` itself. Fixing only the models +leaves a deleted provider coming back, which then makes its models legitimate +again — the ghost returns by a different door. + +**Correction:** the acceptance test asserts both. After an external provider +deletion and an unrelated long-lived-server save, `config.providers[P]` **and** +P's custom models must both stay absent. If whole-array reconciliation cannot +deliver that, the answer is provider-aware reconciliation or a field-scoped +persistence path, not a narrower test. + +## B2 — I described the baseline mechanism wrongly + +I wrote that the baseline is "set when the config is loaded, exactly as +`claudeCodeBaseline` is". That is not what `claudeCodeBaseline` does. It is a +`WeakMap` armed **explicitly by `startServer`** (`src/config.ts:2497`, +`src/server/index.ts:485`), and the comment there says arming is eager on +purpose because lazy arming would lose the hand edit the guard exists to +protect. + +The consequence matters: a CLI process that loads a config and saves it is +**not armed**, so a guard modelled on this would be silently inert there. That +is defensible for `claudeCode`, whose contested writer is the long-lived +server, but it must be a stated policy rather than an accident. + +**Correction:** the plan must name every writer path, state whether it is +armed, and define the unarmed behaviour explicitly. My current position: an +unarmed config has no baseline, so it cannot claim "I did not change this" — +the safe default there is the provider-absence prune from B3, which needs no +baseline at all. That is why the prune is not optional. + +## B4 — deletion is consistent, but the marker needs its own test + +The reviewer confirmed WP16-B's premise: dependent combos are already rejected +before provider deletion, rename already rewrites `customModels[].provider`, +and `src/server/management/model-rows.ts:55` renders custom rows without +checking that the provider exists. So deletion matches existing lifecycle +behaviour. + +**Correction:** add a regression that deletion removes the visible row *without +corrupting* `legacyOwnedSlugs` in the `customModelCatalogMigration` marker. That +marker grants one-time ownership of pre-marker rows; if deletion silently +rewrites it, an older binary's view of ownership changes, and the migration +file's own comment warns against exactly that. + +## B5 — do not imply #1290's CI was green + +I recorded #1290 as landed without qualifying its CI. Checked: the +Cross-platform run at its head `0fe140f91` concluded **`cancelled`**, not +`success`. + +What actually validates the fix is `dev` afterwards. `dev` at `5222f354a` also +shows `cancelled`, and the first `success` on `dev` after it is `57ea8df47` — +the #1288 merge, which contains #1290's change as an ancestor. So the Grok +weekly fix *is* covered by a green `dev` run, but by inheritance, one merge +later, and not at its own head. + +That distinction is worth stating rather than smoothing over: "it's on `dev` +and `dev` is green" is a weaker claim than "its own CI passed", and the run +ids are what let a reader tell which one they are being given. The same +`test 3/4` flake recorded in WP15 is the likely cause of both cancellations. + +## Revised acceptance criteria + +1. #1283 closed with the landed SHA, the manual-close reason, **and** the + honest CI provenance from B5. +2. #1279's CI approved at a SHA-matched head, logged, disposition recorded. +3. Both #1273 defects fixed. Defect 2's fix reconciles per row and prunes rows + whose provider is absent; it does **not** rely on whole-array comparison. +4. Regressions, each with its own ablation: + a. provider removal drops that provider's custom models; + b. stale save does not resurrect deleted rows; + c. **disk deletes P while memory edits Q** — Q survives, P does not; + d. stale save does not resurrect the deleted **provider** either; + e. deletion leaves `legacyOwnedSlugs` intact. +5. Every writer path named with its arming status, and the unarmed policy + stated. +6. Full suite, typecheck, privacy scan green on the rebased branch. +7. PR against `dev`, all three template sections, reporter credited. + +## Writer-path survey, as B2 requires + +`saveConfigPreservingClaudeCode` / `validateAndSave` are called from about +twenty files: + +``` +11 src/server/management/agent-settings-routes.ts + 7 src/server/management/provider-routes.ts + 5 src/server/management/oauth-account-routes.ts + 5 src/providers/api-keys.ts + 5 src/cli/claude-desktop.ts + 4 src/server/management/routing-profile-routes.ts + 4 src/server/management/config-routes.ts + 4 src/cli/provider.ts + 3 src/server/management/combo-routes.ts + 3 src/codex/routing.ts + ... 10 more +``` + +They split into two populations: + +- **Management routes** (`src/server/management/*`) run inside the long-lived + server, which armed the baseline at `startServer`. A per-config baseline + guard works there. +- **CLI paths** (`src/cli/provider.ts`, `src/cli/claude-desktop.ts`, + `src/cli/models.ts`) are short-lived processes that load, mutate, and exit. + They are **never armed**, and arming them would be meaningless: they read + disk moments before writing, so their "baseline" is the disk. + +This settles the B2 policy and reinforces B3. A baseline-only guard is inert +across roughly half the call sites, so correctness cannot rest on it. The +provider-absence prune needs no baseline and therefore holds on every path, +which is why it is the load-bearing half of the fix and the row-keyed +reconciliation is the refinement layered on top where a baseline exists. + +A guard that silently does nothing on half its call sites is the kind of fix +that reads well in a diff and fails in the field — which is the shape of the +original defect, where two sibling operations disagreed about the same array. + +--- + +# Second audit fold — the design was still wrong in four places + +A second review round returned `VERDICT: fail` again. B5 is closed; B1–B4 are +not, and two of them invalidate the replacement design rather than refining it. + +## B2 — `routedSlug` is an encoding, not an identity + +This is the worst error in the plan so far, because it was introduced *by* the +fix for the previous worst error. I keyed row reconciliation on +`routedSlug(provider, modelId)`. Both components are mutable: + +- `PUT /api/custom-models/:id` accepts a new `modelId` + (`src/server/management/model-routes.ts:356` — it looks the row up by + `cm.id === id` and then reassigns `cm.modelId`). +- provider rename rewrites `model.provider` + (`src/providers/provider-id-rewrite.ts:97`). + +So a renamed row looks like a *deleted row plus a new row* under my key. Three- +way reconciliation would then either drop the rename or keep both copies. The +reviewer's phrase is the right test: one row must survive, not two conflicting +copies and not zero. + +The type already carries the right key. `OcxCustomModel.id` is a +`crypto.randomUUID()` assigned at creation (`src/types.ts:531`–`:533`) and never +rewritten by rename or by the PUT — which is exactly why the PUT route looks +rows up by it. + +**Correction:** reconcile keyed by `OcxCustomModel.id`, with an explicit +field-level conflict policy: a row present on both sides takes the in-memory +field values where memory differs from baseline, and disk values otherwise. A +row absent on disk and unchanged in memory is a remote delete and stays deleted. +Regressions must cover a disk-side provider rename and a disk-side `modelId` +change, each concurrent with an in-memory metadata edit of the same row. + +I should have found this myself: I *read* the PUT route while confirming defect +1 and still reached for the slug, because the slug is what the catalog uses. +Catalog-facing identity and storage identity are different things. + +## B1 — the provider record itself is still unreconciled + +I widened the acceptance criterion to require that a stale save resurrect +neither the provider nor its rows, then wrote a design that only reconciles +`customModels`. Pruning models cannot make `providers[P]` absent, and +`saveConfigPreservingClaudeCode` still serializes the whole candidate object. +The criterion and the design contradict each other, and the criterion is right. + +**Correction:** the design must name how `providers` is reconciled. Two options, +to be decided with evidence rather than taste: + +1. Extend keyed reconciliation to `providers` — remote deletes win when the + in-memory record is unchanged from its baseline. +2. A field-scoped persistence operation: callers declare which top-level fields + they are changing and only those are written, leaving everything else at the + on-disk value. + +Option 2 fixes the entire class rather than two fields, but it changes every +call site and is a much larger blast radius; option 1 keeps the change local at +the cost of leaving the next field to be discovered the same way `customModels` +was. The decision needs the call-site matrix below to be made honestly, and it +is explicitly *not* made in this revision. + +## B3 — the writer survey was approximate, so its conclusion was unearned + +I wrote "…10 more" and then drew a two-population conclusion from a list I had +truncated. The reviewer named a counterexample I had elided: +`src/storage/policy.ts:265` loads an unarmed config and calls the wrapper, and +it is neither a management route nor a CLI command. + +Exhaustive matrix, all 13 files that call `saveConfigPreservingClaudeCode` or +`validateAndSave`, with how each obtains its config: + +| File | calls | config provenance | +|------|-------|-------------------| +| `src/server/management/agent-settings-routes.ts` | 8 | `loadConfig()` | +| `src/providers/api-keys.ts` | 4 | passed-in | +| `src/cli/provider.ts` | 4 | `loadConfig()` | +| `src/cli/claude-desktop.ts` | 4 | `loadConfig()` | +| `src/server/management/oauth-account-routes.ts` | 4 | passed-in | +| `src/server/management/config-routes.ts` | 3 | `loadConfig()` | +| `src/server/management/combo-routes.ts` | 2 | passed-in | +| `src/codex/routing.ts` | 2 | passed-in | +| `src/server/management/provider-routes.ts` | 1 | passed-in | +| `src/storage/policy.ts` | 1 | `loadConfig()` | +| `src/codex/auth-api.ts` | 1 | `loadConfig()` | +| `src/providers/key-failover.ts` | 1 | passed-in | +| `src/config.ts` | 1 | internal | + +The honest conclusion is not "two populations". It is that **provenance is +mixed within every layer**: management routes both load fresh and mutate a +long-lived object, and non-route modules (`storage/policy`, `codex/routing`, +`providers/key-failover`, `providers/api-keys`) write config too. A guard keyed +to `startServer` arming covers some of these and not others, and which is which +is not predictable from the directory. + +## B4 — the prune needs a stated precondition, not universal application + +I claimed the provider-absence prune is safe everywhere because it needs no +baseline. The reviewer's objection stands: `saveConfigPreservingClaudeCode` +performs no runtime full-config validation, and `auth-api.ts:377` treats any +object with a truthy `providers` as a runtime config. Nothing structurally +prevents a caller from saving a filtered or partially built config, and a prune +would silently delete that user's rows. + +The reviewer looked and found no production writer that deliberately saves a +partial config — but "I could not find one" is not an invariant, and my plan +asserted safety without establishing one. + +**Correction:** the prune applies only to a config proven to carry an +authoritative provider map. Either enforce that precondition at the write +boundary explicitly, or scope the prune to the reconciled snapshot the write +path itself builds from the on-disk read. A test must show that a filtered or +partial caller cannot silently delete retained rows. + +## Where this leaves WP16 + +Two rounds of review have found, in order: a rule that resurrects deleted rows, +a key that duplicates renamed rows, a criterion contradicting its own design, a +truncated survey used to justify a conclusion, and an unproven safety claim. +That is a defect whose correct fix is a genuine concurrency design, not a patch +I can land credibly inside this session's remaining scope. + +**Disposition: #1273 stays open with a documented diagnosis rather than a rushed +fix.** Both defects are confirmed at file:line and that is real value for +whoever picks it up. Shipping my third design attempt — after two were shown to +lose user data — into a config-persistence path would be the least defensible +thing in this entire campaign. + +The diagnosis goes on the issue: both defect sites, the asymmetry with +`claudeCode` and the server binding, the rename-vs-remove inconsistency, the +identity requirement (`OcxCustomModel.id`, not the slug), the writer matrix, and +the two candidate designs with their tradeoffs. + +--- + +# Third audit fold — the matrix was still wrong, and the hold was too wide + +## The "exhaustive" matrix was not exhaustive + +I built it by grepping the wrapper name, which misses every aliased binding. +The management routes bind it through a test-injection seam: + +``` +model-routes.ts:129 const persistConfig = deps.saveConfigPreservingClaudeCode ?? saveConfigPreservingClaudeCode; +routing-profile-routes.ts:316 const save = deps.saveConfigPreservingClaudeCode ?? saveConfigPreservingClaudeCode; +native-integration-routes.ts:731 const persist = deps.saveConfigPreservingClaudeCode ?? saveConfigPreservingClaudeCode; +``` + +So `model-routes.ts` — the file that owns custom models — showed up as **zero +writes** in a matrix I published as exhaustive, in a document arguing that the +previous version's approximation was the problem. Corrected inventory, direct +plus aliased: + +| File | direct | aliased | +|------|--------|---------| +| `src/server/management/agent-settings-routes.ts` | 8 | 2 | +| `src/server/management/provider-routes.ts` | 1 | 7 | +| `src/server/management/model-routes.ts` | 0 | 6 | +| `src/cli/claude-desktop.ts` | 4 | 0 | +| `src/cli/provider.ts` | 4 | 0 | +| `src/providers/api-keys.ts` | 4 | 0 | +| `src/server/management/oauth-account-routes.ts` | 4 | 0 | +| `src/server/management/config-routes.ts` | 3 | 0 | +| `src/server/management/combo-routes.ts` | 2 | 0 | +| `src/codex/routing.ts` | 2 | 0 | +| `src/server/management/routing-profile-routes.ts` | 0 | 2 | +| `src/server/management/native-integration-routes.ts` | 0 | 1 | +| `src/storage/policy.ts` | 1 | 0 | +| `src/codex/auth-api.ts` | 1 | 0 | +| `src/providers/key-failover.ts` | 1 | 0 | +| `src/config.ts` | 1 | internal | + +**16 writer files; 20 wrapper references.** The distinction matters and I +blurred it: `src/server/management-api.ts`, `management/context.ts`, +`management/logs-usage-routes.ts`, and `management/shared.ts` import or +type-reference the wrapper without ever invoking it (`context.ts:21` declares it +as an optional dependency for the test seam). Counting those as writers +overstates the surface, which is the same species of error as the undercount it +replaced — I fixed a number by changing it to a different wrong number. + +Also corrected: I labelled `auth-api.ts` as `loadConfig()`-only, but +`getRuntimeConfig` at `:381` prefers a passed-in runtime config when one is +supplied. + +The conclusion survives and strengthens — provenance is mixed and arming-keyed +guards are inert on much of this surface — but I published a false count while +criticising an approximate one. Both public comments on #1273 were corrected. + +## Splitting the disposition: defect 1 ships, defect 2 holds + +The reviewer rejected the all-or-nothing hold, correctly. My reasoning was "a +stale save can reintroduce the rows anyway, so cleaning them up is theatre". +That conflates two things: defect 2 is a *concurrency* defect that needs a +reviewed persistence design, while defect 1 is a *lifecycle* gap in two direct +code paths that already do provider cleanup and already reject dependent combos +first (`provider-routes.ts:604`). One being unfinished does not make the other +unsafe or useless. + +**Revised disposition:** + +- **Defect 1 — fix now.** Filter `config.customModels` by provider in both + removal paths, with a regression proving `legacyOwnedSlugs` in the + `customModelCatalogMigration` marker is not corrupted. +- **Defect 2 — stays open on #1273** with the diagnosis, pending a persistence + design reviewed on its own terms. + +This is the third time in this work-phase that the review changed my answer +rather than polishing it, which is the argument for running the gate at all. + +--- + +# Fourth audit fold — the CLI half had no test + +Four more blockers, all accepted. + +**The one that mattered: I fixed the CLI path and never tested it.** The +ablation I presented reverted *both* files at once and showed one API test +failing, which proves the management wiring and says nothing about +`src/cli/provider.ts`. A reviewer reading "ablation passes" would reasonably +assume both halves were covered. They were not. + +Added `tests/cli-provider.test.ts` → "provider remove drops that provider's +custom models (#1273)", which spawns the real CLI, asserts the persisted +`config.json`, and asserts the new `--json droppedCustomModels` field. Ablating +`src/cli/provider.ts` **alone** now gives 29 pass / 1 fail, and that failure is +the CLI test. Two paths, two independent proofs — which is what the previous +work-phase already established as the standard and what I failed to apply here. + +**Marker persistence was asserted on the helper, not the write path.** The +marker test only proved `dropProviderCustomModels` does not mutate the in-memory +object. It never exercised `projectCustomModelCatalogMigration`, which runs +inside the save. Both integration tests now seed +`customModelCatalogMigration` and assert its value in the persisted +`config.json` after the delete, so the claim covers the real path. + +**A false comment in the shipped code.** I wrote that dropping the emptied key +leaves a config "byte-identical to one whose last custom model was removed" — +untrue, because the migration marker deliberately survives. Narrowed to the +claim actually being made: the `customModels` field is absent either way. + +Final verification on `57ea8df47`: + +- `bun run test` — **10008 pass / 7 skip / 0 fail**, 626 files +- `bun test` on the three touched suites — 62 pass, then 30 pass for the CLI suite +- ablation, management path only — 61 pass / 1 fail (the API test) +- ablation, CLI path only — 29 pass / 1 fail (the CLI test) +- `bun run typecheck` clean, `bun run privacy:scan` passed + +--- + +# Outcome + +| Item | Disposition | +|------|-------------| +| #1283 grok weekly limit | **closed** — fixed on `dev` by #1290 (`5222f354a`), closed manually with a public correction about its CI provenance | +| #1273 defect 1 (orphaned rows) | **fixed** — PR #1293, `472015e5a` | +| #1273 defect 2 (stale whole-document write) | **open with a diagnosis**, deliberately not patched | +| #1278 / #1279 Windows console flash | **awaiting CI** — approved at a SHA-matched head, three shard hangs so far, diagnosed publicly | + +## #1279 and the shard that keeps hanging + +Three Cross-platform runs on #1279 ended `cancelled`. Each time shard +`test 2/4` (or `3/4`) runs normally, then stops emitting output entirely until +the runner kills it roughly 14 minutes later. Nothing fails. + +The same pattern hit `dev` with no PR involved (`31259450263`, +`31259447622`) and cost #1288 two reruns before `rerun-failed-jobs` went green, +so the base branch has an unstable shard today. That is the likely answer. + +I did not simply write "flake" on the PR and move on, because #1279 changes +process and identity lookups and a hang right after a proxy server starts is the +shape a blocking child-process call would take. What argues against it: the +hanging shard runs on Linux, where the `windows-*` modules should never be +reached. I have no Windows host, so I told the author exactly that — what the +evidence shows, what I cannot rule out, and the one concrete thing worth +checking (timeouts on lookups reachable during startup) — rather than either +dismissing it or implying their patch is at fault. + +## The pattern across WP16 + +Four review rounds, and each one changed the answer rather than polishing it: + +1. Whole-array reconciliation → resurrects a deleted provider's rows. +2. `routedSlug` as a row key → duplicates renamed rows. I had *read* the PUT + route that mutates `modelId` and still reached for the slug, because the slug + is what the catalog uses. Catalog identity and storage identity are different + things. +3. "Exhaustive" writer matrix → missed every aliased binding, including the file + that owns custom models. +4. All-or-nothing hold → wrong; defect 1 was bounded and shippable, and holding + it gained nothing. + +Plus two false public claims I had to retract on the issue: a wrong writer count +(13), then a differently wrong one (20 references, 16 writers). + +The useful lesson is narrower than "review is good". Every one of these was a +claim I could have checked and did not, because the claim felt like background +detail rather than the thing being decided. The slug key and the writer count +were both stated in passing while my attention was on the reconciliation rule. diff --git a/devlog/_plan/260808_bug_campaign/025_wp3_lane_c_ci_workflows.md b/devlog/_plan/260808_bug_campaign/025_wp3_lane_c_ci_workflows.md new file mode 100644 index 000000000..dffc170a9 --- /dev/null +++ b/devlog/_plan/260808_bug_campaign/025_wp3_lane_c_ci_workflows.md @@ -0,0 +1,164 @@ +# WP3 — lane C: the CI/workflow stack + +Three PRs, and the interesting result is that the two open ones needed opposite +treatment despite looking similar on the board. + +| PR | State entering WP3 | Outcome | +|----|--------------------|---------| +| #1255 harden comment-driven review workflows | merged (`0993c53ae`) | already done | +| #1185 bind Windows shard assertion | draft, CI **failure**, 324 behind | **republished** as #1301 | +| #1259 fail-closed aggregate-check evidence | draft, CI `cancelled`, 20 behind | **held** with a blocker | + +## #1185 — a red PR that was right + +Its Cross-platform CI at `bff31d1e0` genuinely failed, which is the kind of +signal that gets a stale draft closed. It should not have here. + +The PR touches exactly one file, `tests/ci-workflows.test.ts`, and that file +only reads workflow YAML as text. The crash was somewhere else entirely: + +``` +##[group]tests/autostart-health.test.ts: +# Unhandled error between tests +error: EEXIST: file already exists, epoll_ctl + at new WriteStream (internal:fs/streams:244:58) +# then +error: Cannot call describe() after the test run has completed + at tests/autostart-health.test.ts:23:1 +``` + +2142 pass / 1 fail / 2 errors. A Bun-level failure while loading a file the +diff cannot reach, with the `describe()` error as collateral. + +My first write-up called this "an fd leak from a preceding test file". The +audit removed that: the log shows *where* the crash happened, not *why*, and I +had asserted a mechanism the evidence does not carry. Recorded as an unrelated +Bun/runner load failure, root cause unknown. + +### What the patch actually buys + +The existing assertion used `.includes()` on the Windows step's `run` text, so +the command counted as present anywhere in the script — inside an `echo`, or in +a comment. Measured on current `dev` by mutating `.github/workflows/ci.yml:492` +so the Windows leg prints instead of runs: + +| Mutation | `dev` today | with #1185 | +|----------|-------------|------------| +| `run: bun test …` → `run: echo bun test …` | 125 pass / 0 fail | 124 / **1 fail** | +| add `if: false`, command unchanged | 125 pass / 0 fail | 124 / **1 fail** | + +The second row is mine. The audit pointed out that binding the assertion to an +executable *line* still permits an unreachable *step*: the exact command under +`if: false` runs nothing and satisfies the contributor's check. Both mutations +leave `dev` green today, which is the whole argument for landing this. + +Published as #1301 with the two commits separated — `364b358` carries luvs01's +`Co-authored-by`, `f09ef15` is mine with no trailer and is called out in the PR +body. + +## #1259 — the right idea with a hole in its central claim + +#1259 removes `pull_request.paths` and moves scope gating into the existing +`changes` job, so a docs-only PR gets an explicit passing `ci` check instead of +no check at all. That problem is real: no check is harmless until the check +becomes required, and then it is a PR that waits forever. + +The blocker is in the property the PR is named for. `changes` exposes +`ci: ${{ steps.filter.outputs.ci }}` with no validation, and every expensive job +gates on `needs.changes.outputs.ci == 'true'`. If `changes` **succeeds** while +that output is empty or malformed — an action upgrade renaming an output, a +filter-syntax slip — then: + +1. every expensive job evaluates `'' == 'true'` and is skipped; +2. the aggregate gate treats `skipped` as a pass, deliberately, because that is + how it recognises trigger-scoped jobs; +3. `ci` reports green having tested nothing. + +`changes` *failing* is handled — the aggregate catches it. It is `changes` +succeeding with an unusable output that slips through, and today's `paths:` +trigger makes that unreachable, so the PR turns a non-issue into the single +point of truth without hardening it. + +I suggested a validation step on the PR — and got it wrong on the first pass by +writing `case "${{ steps.filter.outputs.ci }}"`, interpolating the expression +straight into shell. That is the injection shape this repository's workflow +hardening exists to prevent. Low risk from a SHA-pinned action, but wrong, and +corrected publicly to pass the value through `env:` so bash sees data. + +I also claimed #1265 and #1259 would conflict in `enforce-pr-target.yml`. I had +compared branch positions, not hunks. Corrected to "may conflict; decide the +integration order". + +**Disposition: held, not approved.** It changes when CI runs at all, which +`MAINTAINERS.md` puts in the security-review class, and I offered to implement +the validation step rather than making the author respin. + +One thing the audit checked that I had not: whether removing the path filter +widens exposure on the self-hosted Windows runner. It does not — PR Windows +stays `workflow_dispatch`-only. But it does make the aggregate check +security-critical, which is exactly why the output needs validating. + +## Faults recorded + +- Asserted a mechanism (fd leak) the log did not support, when "root cause + unknown" was the honest reading. +- Suggested a workflow snippet with an expression-injection shape while + reviewing a security-class change. +- Claimed a conflict from branch divergence without looking at the hunks. + +--- + +# The "flake" I called five times + +#1301's CI came back `cancelled` with `test 4/4` hung. I issued +`rerun-failed-jobs`, as the four-state rule says, and asked the reviewer whether +I was now pattern-matching to "flake" too readily. The answer was yes, with a +detail I had not checked: attempt 1 was **not** a superseded run. `test 4/4` ran +its Test step for a full 15 minutes and was killed by the job timeout. + +The retry then did the same thing — 15:28:02Z to 15:43:17Z, cancelled at 15 +minutes 15 seconds. Two real timeouts at the same head. + +So I stopped rerunning and investigated instead. The shape is identical every +time: output stops immediately after a test that starts a proxy listener, +silence for ~14 minutes, then `Terminate orphan process: pid (NNNN) (bun)` in +cleanup. In #1301 the last line was + +``` +[web-search-loop] cancelled — 1 real searches, 0 placeholders, 13ms +(pass) routed Claude requests give OpenAI sidecars main auth without leaking it to the routed provider +``` + +from `tests/claude-messages-endpoint.test.ts` — which passes locally in 2.7s +(38/38), and the full suite is 10009 pass. The stall is *after* the assertion, +so teardown or the next file's setup is the suspect, not the test. + +Five occurrences today across four unrelated branches **and `dev` itself**: + +| Run | Branch | Shard | +|-----|--------|-------| +| 31263738953 | `codex/260808-1185-windows-shard-assertion` | `test 4/4`, twice | +| 31255199569 | `fix/windows-powershell-popup` | `test 2/4` | +| 31258815611 | `codex/260808-1195-unbound-quota-unknown` | `test 3/4` | +| 31152916419 | `agent/test-windows-ci-shard-command` | `test 3/4` | +| 31259450263, 31259447622 | `dev` | various | + +The varying shard argues against one bad test. The one instance that did not +hang is the clue: it crashed with `EEXIST: file already exists, epoll_ctl` in a +Bun `WriteStream` — a descriptor registered with the event loop twice, which is +the same resource-lifecycle fault a deadlocking registration would produce. + +Filed as **#1302** with the run inventory, and #1301 is **held** rather than +rerun to green. + +## Why this is the fault worth recording + +Three of the four cancelled runs *did* go green on retry, so "rerun until green" +worked every time and produced merges I still stand behind. The problem is that +it works equally well on a genuine hang introduced by a real change. I applied +the rule correctly — `cancelled` means rerun — and used it to avoid looking at +five instances of the same failure. + +What broke the loop was being asked to justify the call rather than state it. +"It's flake" was a conclusion I never had evidence for; I had evidence that +retrying made it go away, which is a different claim. diff --git a/devlog/_plan/260808_bug_campaign/026_wp5_large_solo.md b/devlog/_plan/260808_bug_campaign/026_wp5_large_solo.md new file mode 100644 index 000000000..bee7f7b55 --- /dev/null +++ b/devlog/_plan/260808_bug_campaign/026_wp5_large_solo.md @@ -0,0 +1,94 @@ +# WP5 — the two large solo PRs + +| PR | Size | Outcome | +|----|------|---------| +| #1244 preserve routed models in desktop picker | 58 files | **CI green, merge held on four conditions** | +| #1228 native image support for Cursor | 8 adapter files | **held for the author**, conflicting and stale | + +## #1244 — the author's fix was better than the one I proposed + +In WP15 I reported the CI failure at `15545b3d1`: `TypeError: +suppressedBareNativeSlugs.has` at `src/codex/catalog/sync.ts:427`, driven by +`tests/codex-v2-gate.test.ts:1211`, with the input destructured at `:385` +without a default. I named two fix shapes — default the input, or update every +caller — and left the choice to them. + +They did neither. `2c9994a9d` adds the two missing sets to the **one test call +site**, two lines, and leaves `sync.ts` alone. + +That is the correct fix, and my reasoning about why was also wrong. I told the +audit that required-field typing beats a default because `typecheck` enforces +every caller. The audit checked what I had not: + +``` +$ rg '"include"' tsconfig.json +15: "include": ["src"] +``` + +**`bun run typecheck` never covered `tests/` at all.** So my "typecheck is the +caller-sweep proof" claim was empty — the failing site was in the one directory +the compiler does not read, which is exactly why it reached CI. The right +statement is narrower: the fields are genuinely required on +`ObservedCatalogEntryBuildInput`, production callers in `src/` are compiler- +checked, and a hand-built test literal is the residual gap that explicit empty +sets close honestly. + +I had reached for "the author saved me work and was therefore right". The +conclusion survived; the argument for it did not. + +### What I verified before holding + +- Cross-platform CI **success** at `2c9994a9d` (run `31258863895`) +- full `bun run test` on that head — **10003 pass / 7 skip / 0 fail**, 623 files +- `bun run typecheck` clean +- catalog/convergence suites — 258 pass / 0 fail +- no semantic conflict with the merged #1212: `a4878de38` is an **ancestor** of + #1244's merge base, so the convergence work is already underneath it +- of the 20 commits `dev` moved ahead, none touch #1244's source files; the only + overlap is `docs-site/.../configuration/routing.md` + +### Why it is still held + +1. **Stale base and stale claims.** 20 behind, and the PR body still asserts + "0 commits behind" and "remains draft" while the PR is non-draft. The + evidence needs to exist at the head that would actually merge. +2. **A locale defect.** English documents that `-` clears `--effort`, + `--alias`, `--display-name` and that the subcommands exist under + `ocx route combo` (`guides/combos.md:264`). Russian omits both + (`ru/guides/combos.md:224`); `ja`, `ko`, `zh-cn` carry them. Five-locale + alignment was claimed, four are aligned. +3. **CI evidence standard, post-#1302.** One green run is currently weaker + evidence than it looks, so a 57-file catalog change gets two completed + non-cancelled runs at the same rebased SHA. + +4. **Fresh activation evidence.** The description's screenshot is carried + forward from #1056, but this branch is a reconstruction rather than that + code, so it proves nothing about what would merge. A picker capture at the + current head is a **condition**, not a request — I softened it to "a request + rather than a blocker" in the first draft of this page, which quietly + downgraded something the review had made a merge condition. + +## #1228 — where the republish protocol stops + +Conflicting, draft, untouched since 2026-08-07, four readiness boxes unticked. +Every other stale PR in this campaign got rebased and republished for the +author. This one did not, and the line is worth stating because it is the same +line WP15 crossed for #1244 and then had to retreat from. + +The republishes were small and mechanical — a net diff that reapplies onto a +moved base with the author's intent unambiguous. #1228 adds native image +support across eight files of the Cursor adapter including the protobuf request +builder and live transport. Resolving those conflicts means re-deciding the +author's design against a base that moved underneath it, which is authorship, +not maintenance. + +Told them so directly, offered to close it as stale if they would rather not +carry it, and noted that a `cancelled` shard is #1302 and mine to chase rather +than theirs. + +## Fault recorded + +Claimed `typecheck` proved a caller sweep it structurally cannot perform, +because `tsconfig.json` includes only `src`. I have run that command dozens of +times this session and cited it as evidence repeatedly; I had never read what it +covers. diff --git a/devlog/_plan/260808_bug_campaign/027_wp4_lane_d_close.md b/devlog/_plan/260808_bug_campaign/027_wp4_lane_d_close.md new file mode 100644 index 000000000..8cff9cd0e --- /dev/null +++ b/devlog/_plan/260808_bug_campaign/027_wp4_lane_d_close.md @@ -0,0 +1,152 @@ +# WP4 — closing lane D's catalog sequence + +The plan in `040_wp4_catalog_sequential.md` ordered seven PRs so each landed on +a `dev` the previous one had already moved: `#1224, #1226, #1178, #1266, #1244, +#1163, #1228`. All seven now have a disposition; two of those dispositions are +"waiting on something specific" rather than closed, and the difference is +recorded per row instead of flattened into "terminal". + +| PR | Disposition | +|----|-------------| +| #1224 | merged (`903b69b4b`) | +| #1226 | merged (`3ad5bb6bd`) | +| #1266 | merged (`28ba79377`) | +| #1178 | **merged** as `e8ec8d191` — but without a recorded approval, see below | +| #1244 | CI green, **held** on three conditions (WP5) | +| #1163 | **closed as superseded**; republished as #1305, **merged** as `794d8eb09` | +| #1228 | **held for its author** (WP5) | + +## #1178 — the author fixed it themselves + +In WP14 I diagnosed a cache-invalidation defect here and built a fix on a local +branch, then offered it on the PR rather than pushing it. Their head is now +`2ebdb705c fix(catalog): distinguish cache eviction from authority changes` — +they wrote it themselves. Offering rather than pushing was the right call, and +the local branch `codex/260808-1178-cache-clear-reason` can be abandoned. + +Approved its `action_required` CI at a SHA-matched head; it came back +`success`. + +### Two corrections I had to publish on that PR + +**I told the author they were waiting on their own checklist.** They were not: +`isDraft` is false and all four readiness boxes are ticked. The PR is waiting +on *maintainer approval* — `reviewDecision` is empty — which is my side, not +theirs. Telling a contributor the ball is in their court when it is in mine is +the specific failure mode the readiness gate exists to avoid. + +**And my token-path description named the wrong function.** See below. + +### Security review, published rather than implied + +The audit caught that approving a CI *action* is not the security review +`MAINTAINERS.md:48` requires, and this PR touches OAuth token retrieval and +account authority. So I did the review and posted it: + +- **Token flow.** I originally credited `getValidAccessTokenSnapshot`. That is + one route; the catalog gather running on filesystem evidence goes through + `observedModelsAuthResolver` → `observeActiveOAuthAccessToken` + (`provider-fetch.ts:821`). The property holds either way — captured + synchronously before any await, passed only as `apiKey` into the request + builder — but the observed path also carries a credential identity and a + cache generation, so a token that changes underneath cannot be attributed to + the earlier gather. That guard is more specific than what I credited. +- **A guard I first missed, then over-credited.** `modelDiscoveryTransportSeed` + (`oauth/index.ts:614`) pins the registry's fixed `baseUrl` and adapter for + OAuth presets *before* the Bearer header is materialized, so a hand-edited + `config.baseUrl` cannot receive an OAuth token. I omitted it from the first + review, then called it the headline — and it **already existed at the merge + base `3ad5bb6bd`**. #1178 neither added it nor repaired an arbitrary-host + leak. The accurate claim is smaller: the new CCA POST discovery path inherits + the existing pin rather than bypassing it. + +## The governance failure + +`MAINTAINERS.md:45` requires a maintainer approval **and** green required CI +before merge. Six PRs went to `dev` today with green exact-head CI and **no +recorded `APPROVED` review**: #1287, #1288, #1289, #1293, #1305, #1178. + +The mechanism is worth naming because it is not simple forgetfulness. Several +of these needed a pending Actions run approved — `action_required`, which +`gh pr checks` hides — and I logged every one of those against its head SHA +into `.tmp/ocx_approval_ledger.tsv`. Doing the careful version of the *wrong* +approval made the missing one feel handled. On #1178 I also published a full +security review as a comment, so the review existed; it just was not an +approval. + +Not back-filling them. A review recorded after the merge it was meant to gate +is a worse artifact than an accurate record of the gap. Filed as **#1306**, +which also notes the real structural hole: `MAINTAINERS.md:47` forbids +approving your own PR, and five of the six were maintainer republishes of +contributor work, so the convention has no defined path for a solo maintainer +landing someone else's rebased patch. +- **Log surface** is where a discovery failure usually leaks. Both new + `console.warn` sites are clean: the Cursor path logs classified error/detail, + and the provider path logs `status`, `contentType`, `fallback`, and + `urlClass` — a two-value hostname classification + (`provider-fetch.ts:975`), not the URL. That matters because Vertex endpoints + embed a project id and a raw URL would carry query parameters. +- **Snapshot-before-await** is also the right ordering for the cache concern: + an OAuth account change mid-flight cannot make a stale-but-valid response + look authoritative for the new account. + +## #1163 — a refused `git apply` that was not a semantic rebase + +366 commits behind, `CONFLICTING`, and `git apply --check` rejected the net +diff outright. That is normally where WP5's line applies and the PR goes back +to its author. + +The actual merge disagreed: **two conflicts, both a single line, both the same +cause.** `dev` had renamed `augmentRoutedModelsWithJawcodeMetadata` to +`augmentRoutedModelsWithMetadata` and added +`CODEX_ACCOUNT_BOUND_CATALOG_KIND` plus a `catalog/parsing` import block; the +branch had added `resolveComboCatalogMember` to the same export and import +lines. Keeping every symbol from both sides resolves it without re-deciding +anything. + +So `git apply` refusing is evidence about *textual* applicability, not about +whether a rebase requires judgement. Running the merge and reading the conflicts +is the cheap check that tells them apart, and skipping it would have sent a +mechanical rebase back to a contributor for no reason. + +### The fault the audit caught + +I reported the resolution as done because the working files had no conflict +markers and the tests passed. The index still held `UU` entries for both files — +git could not have committed that state. Marker-free files are not a resolved +merge, and "the tests pass" was true of a tree that did not exist as a commit. + +Staged both, confirmed `git diff --cached --check` clean, then re-ran the full +suite **on the committed tree**: 10013 pass / 0 fail. Published as #1305. + +The audit also flagged that the PR body ran those two facts together, reading +as though the staged-tree whitespace check were committed-tree evidence. Body +amended to separate them. + +## Consistency check: why #1163 was rebased and #1228 was not + +Both are stale contributor PRs and #1163 is *older* (366 vs ~200 commits), so +the line cannot be age. It is whether integration requires deciding something +the author already decided: + +- **#1163** — two import lines. The contributor's semantics are untouched and + their tests still exercise them. +- **#1228** — eight files of Cursor adapter including the protobuf request + builder and live transport, where resolving conflicts means re-deciding how + their image support interacts with a moved base. + +Age raises the verification bar. It does not decide who owns the merge. + +## Faults recorded + +- Published a review that named the wrong token-resolution function and omitted + the strongest guard in the diff. +- Told a contributor they were waiting on their own checklist when the PR was + ready and waiting on me. +- Wrote "all seven have a terminal disposition" while two were waiting on CI + and on maintainer approval. "Dispositioned" and "finished" are not the same + claim, and the closeout wording flattened them. +- Merged six PRs without the approval `MAINTAINERS.md` requires, while + meticulously logging a different kind of approval. +- Attributed a pre-existing security guard to the PR under review, in a + correction that was itself correcting an omission. From dea62e4959227464abf5ef9774383b8cd185a2c2 Mon Sep 17 00:00:00 2001 From: chrisae9 Date: Sat, 8 Aug 2026 11:45:51 -0400 Subject: [PATCH 28/77] feat(codex): add account picker lifecycle controls --- .../src/content/docs/guides/web-dashboard.md | 14 +- .../content/docs/ja/guides/web-dashboard.md | 11 +- .../ja/reference/cli/providers-accounts.md | 7 +- .../ja/reference/configuration/providers.md | 5 + .../ja/reference/configuration/routing.md | 24 +- .../docs/ja/reference/management-api.md | 21 +- .../content/docs/ko/guides/web-dashboard.md | 11 +- .../ko/reference/cli/providers-accounts.md | 7 +- .../ko/reference/configuration/providers.md | 5 + .../ko/reference/configuration/routing.md | 22 +- .../docs/ko/reference/management-api.md | 21 +- .../docs/reference/cli/providers-accounts.md | 11 +- .../docs/reference/configuration/providers.md | 7 + .../docs/reference/configuration/routing.md | 11 +- .../content/docs/reference/management-api.md | 27 +- .../content/docs/ru/guides/web-dashboard.md | 13 +- .../ru/reference/cli/providers-accounts.md | 12 +- .../ru/reference/configuration/providers.md | 8 + .../ru/reference/configuration/routing.md | 25 +- .../docs/ru/reference/management-api.md | 25 +- .../docs/zh-cn/guides/web-dashboard.md | 12 +- .../zh-cn/reference/cli/providers-accounts.md | 11 +- .../reference/configuration/providers.md | 6 + .../zh-cn/reference/configuration/routing.md | 23 +- .../docs/zh-cn/reference/management-api.md | 24 +- gui/src/codex-account-mutation.ts | 17 + gui/src/components/AddCodexAccountModal.tsx | 3 +- .../components/CodexAccountPickerSetting.tsx | 153 +++++ gui/src/components/CodexAccountPool.tsx | 36 +- .../codex-account-pool-main-card.tsx | 5 +- .../components/use-add-codex-account-oauth.ts | 19 +- gui/src/hooks/useCodexAccountPool.ts | 11 +- gui/src/i18n/de.ts | 9 + gui/src/i18n/en.ts | 9 + gui/src/i18n/ja.ts | 9 + gui/src/i18n/ko.ts | 9 + gui/src/i18n/ru.ts | 9 + gui/src/i18n/zh.ts | 9 + gui/src/pages/CodexAuth.tsx | 2 + gui/src/pages/Providers.tsx | 26 +- gui/src/pages/providers-page-modals.tsx | 3 +- gui/src/styles.css | 20 + gui/src/ui.tsx | 9 +- gui/tests/add-codex-account-oauth.test.tsx | 40 +- .../codex-account-picker-setting.test.tsx | 280 +++++++++ .../codex-account-pool-behaviour.test.tsx | 25 + .../codex-account-pool-toast-tone.test.tsx | 21 + .../providers-codex-completion-toast.test.tsx | 234 ++++++++ src/cli/account-auth.ts | 4 +- src/cli/account-catalog-refresh.ts | 14 + src/cli/account-extended.ts | 16 +- src/codex/account-lifecycle.ts | 14 +- src/codex/auth-api.ts | 256 ++++++-- src/codex/catalog-refresh-status.ts | 87 +++ src/server/management-api.ts | 32 +- src/server/management/config-routes.ts | 100 +++- structure/02_config-and-codex-home.md | 2 +- structure/05_gui-and-management-api.md | 13 +- tests/cli-account.test.ts | 75 ++- tests/codex-auth-api.test.ts | 560 +++++++++++++++++- tests/codex-catalog-refresh-status.test.ts | 109 ++++ tests/helpers/catalog-convergence.ts | 10 +- tests/provider-workspace-auth.test.ts | 6 +- tests/settings-stream-mode.test.ts | 226 ++++++- 64 files changed, 2640 insertions(+), 205 deletions(-) create mode 100644 gui/src/codex-account-mutation.ts create mode 100644 gui/src/components/CodexAccountPickerSetting.tsx create mode 100644 gui/tests/codex-account-picker-setting.test.tsx create mode 100644 gui/tests/providers-codex-completion-toast.test.tsx create mode 100644 src/cli/account-catalog-refresh.ts create mode 100644 src/codex/catalog-refresh-status.ts create mode 100644 tests/codex-catalog-refresh-status.test.ts diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 4ad5fd7c7..896a38bb9 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -131,6 +131,18 @@ and other providers. - **Refresh quotas** re-reads account usage immediately so routing and the account cards use the same values. - Pool request logs use opaque labels such as `p3fa91c`, never account emails. +- **Target a specific Codex account from the model picker** is an explicit opt-in. When enabled, + ordinary supported GPT picker rows are replaced by one entry per public account selector. + Choosing one locks that conversation to the mapped account: it does not rotate, fall back, or + change the active Pool account. The built-in Codex App login has its own selector; generated maps + normally use `main`, with a collision-safe suffix such as `main-2` when needed. Added accounts + receive stable, privacy-safe labels, and existing custom selector labels are preserved. + Existing conversations and saved model selections continue routing. Turning the setting off + hides generated picker entries without deleting accounts, selectors, or exact routes. Plain GPT + model ids continue to use the configured Pool or Direct behavior. +- Account add, remove, and picker-setting changes are saved before the model catalog is refreshed. + If that bounded refresh cannot finish, the dashboard shows an amber success-with-recovery notice; + run `ocx sync` to retry. The account or setting change itself remains saved. The Providers overview separately summarizes Pool-mode usage as a display-only weighted capacity estimate, alongside the effective account's raw quota and the next capacity recovery. See @@ -166,7 +178,7 @@ The GUI is a thin client over the proxy's JSON management API. Useful endpoints | Endpoint | Purpose | | --- | --- | -| `GET` / `PUT /api/settings` | Read settings or toggle Codex autostart. | +| `GET` / `PUT /api/settings` | Read settings or update Codex autostart, stream/memory settings, and account-targeting picker visibility. | | `GET` / `POST /api/github/star` | Read the `gh`-derived star state, or star the repository. The POST is refused with `403` `agent_consent_required` for agent-driven callers without a dashboard session. | | `GET /api/startup-health` | Read secret-free routing, service, shim, and restart-safety diagnostics. | | `POST /api/startup-action` | Install the background service or Codex launcher shim through fixed, allowlisted actions. | diff --git a/docs-site/src/content/docs/ja/guides/web-dashboard.md b/docs-site/src/content/docs/ja/guides/web-dashboard.md index 955b9cad5..c63c7a77e 100644 --- a/docs-site/src/content/docs/ja/guides/web-dashboard.md +++ b/docs-site/src/content/docs/ja/guides/web-dashboard.md @@ -108,6 +108,15 @@ Codex タスクだけに適用され、このオプション自体が委任を 枠のうち最も高い使用率でスコア付けし、Go/Free プランは 30 日枠のみ使います。 - **クォータ更新**はアカウント使用量を即座に再読み込みし、ルーティングと画面のアカウントカードが同じ値を見るようにします。 - プールリクエストログにはメールの代わりに `p3fa91c` のような不透明なラベルを使います。 +- **モデルピッカーで使用する Codex アカウントを指定** は明示的な opt-in です。有効にすると、通常の + GPT picker 項目が公開 account selector ごとの項目に置き換わります。選択した会話はそのアカウントに + 固定され、Pool のローテーションや fallback は行われず、active Pool account も変わりません。組み込みの + Codex App login には専用 selector があり、生成 map では通常 `main`、衝突時は `main-2` のような安全な + suffix が使われます。追加アカウントには安定した privacy-safe label が割り当てられます。 + 既存の会話と保存済みのモデル選択は引き続きルーティングされます。無効にしても account、selector、 + exact route は削除されず、通常の GPT id は従来どおり Pool / Direct で動作します。 +- account の追加・削除と picker 設定は catalog refresh より先に保存されます。refresh が完了できない場合は + amber の回復案内が表示されます。変更自体は保存済みなので、`ocx sync` で refresh を再試行してください。 Providers の概要は、Pool モードの使用状況を表示専用の重み付き容量推定値として別途まとめ、現在の 有効アカウントの生のクォータと次の容量回復も併せて表示します。表示される項目、不完全な対象範囲の @@ -119,7 +128,7 @@ GUI はプロキシの JSON 管理 API を使うシンクライアントです | エンドポイント | 用途 | --- | --- | -| `GET` / `PUT /api/settings` | 設定を読むか Codex 自動起動をオン/オフします。 | +| `GET` / `PUT /api/settings` | 設定を読み、Codex 自動起動、stream/memory、account-targeting picker の表示を更新します。 | | `GET /api/startup-health` | 秘密情報を含まないルーティング、サービス、shim、再起動安全性診断を読み取ります。 | | `GET` / `POST /api/windows-tray` | Windows トレイの導入・表示状態を読み取り、`install`、`start`、`stop`、`uninstall` を実行します。 | | `POST /api/sync` | 共有モデルカタログを再構築し Codex モデルキャッシュを古い状態としてマークします。 | diff --git a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md index 9c6e6edf7..cab525623 100644 --- a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md @@ -180,17 +180,20 @@ preemption が未バインドリクエストを直ちに引き上げます。既 ### `ocx account login|reauth|code|cancel ...` -ヘッドレス シェルからブラウザベースまたは手動コードのアカウント認証を実行します。プロバイダー固有のコマンド形式には `ocx account --help` を使用します。 +ヘッドレス シェルからブラウザベースまたは手動コードのアカウント認証を実行します。プロバイダー固有のコマンド形式には `ocx account --help` を使用します。Codex account login は保存済みでも catalog refresh が保留中なら成功終了し、human output の stderr に固定の `ocx sync` 案内を出します。`--json` は案内を混ぜず、完了 state に `catalogRefreshPending: true` を保持します。 ### `ocx account remove --yes [--json]` この保護された非対話型削除には `--yes` が必要です。削除する前に、ID が存在することが確認されます。 ID が欠落している場合は、DELETE を送信せずに 1 が終了します。メインの Codex App ログインは削除できないため、`remove openai main --yes` は拒否されます。削除後、ファミリーは再度読み取られます。固定された Codex アカウントを削除すると、ピンがクリアされ、自動選択に戻ります。 OAuth は最初に残ったアカウントを昇格させるか、何も報告しません。 API キー プールは、最初に残っているキーを昇格するか、何も報告しません。 `--json` の成功と失敗の形状は次のとおりです。 ```text -{ ok: true, provider, id, removedActive: boolean, promotedActiveId: string | null } +{ ok: true, provider, id, removedActive: boolean, promotedActiveId: string | null, catalogRefreshPending?: boolean } { error: string } // stderr, exit 1 ``` +`catalogRefreshPending` は Codex 削除だけに含まれます。`true` でも削除は保存済みで、human output は +stderr に `ocx sync` の案内を出して終了コード 0 のままです。OAuth account と API key の削除形状は変わりません。 + ### `ocx account add-key [--label