From 667ad08f32f204675d2b6efe0eb2338034e1bb15 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 01:56:47 +0800 Subject: [PATCH 01/29] feat(proxy): opt-in same-target 429 wait-and-retry before key failover (#487) Codex never retries HTTP 429 (openai/codex#30471 keeps retry_429=false and the misleading 'exceeded retry limit' error), and single-key pools have no failover, so provider-level retryOn429 waits (Retry-After or fixed interval) and replays the identical pre-stream request on the same key before any key rotation. - types.ts: RateLimitRetryPolicy + OcxProviderConfig.retryOn429 - config.ts: lenient zod validation (strip unknown; typo degrades, never rejects config) - key-failover.ts: rateLimitRetryPolicyFor + rateLimitRetryDelayMs (Retry-After capped at maxIntervalMs) - responses/core.ts: recovery-loop replay before multi-key failover; abort-aware sleep; covers Responses, chat completions, and routed Claude messages - usage/log.ts: AttemptRecoveryKind 'rate-limit-429' - docs-site configuration reference + structure/04 transport note + devlog plan unit - tests: policy unit tests + e2e (single-key replay, passthrough without knob, exhaustion, retry-before-failover ordering) Verified: typecheck, privacy scan, 43/43 retry-related tests; full suite failures are pre-existing on pristine dev in this environment (WS/auth/connection-refused) plus missing-gui-deps artifacts that pass once installed. --- .../000_research.md | 39 +++ .../010_design.md | 53 ++++ .../content/docs/reference/configuration.md | 1 + src/config.ts | 7 + src/providers/key-failover.ts | 46 +++- src/server/responses/core.ts | 45 +++- src/types.ts | 26 ++ src/usage/log.ts | 1 + structure/04_transports-and-sidecars.md | 8 + tests/rate-limit-retry.test.ts | 60 +++++ tests/server-rate-limit-retry-e2e.test.ts | 236 ++++++++++++++++++ 11 files changed, 519 insertions(+), 3 deletions(-) create mode 100644 devlog/_plan/260802_429_same_target_retry/000_research.md create mode 100644 devlog/_plan/260802_429_same_target_retry/010_design.md create mode 100644 tests/rate-limit-retry.test.ts create mode 100644 tests/server-rate-limit-retry-e2e.test.ts diff --git a/devlog/_plan/260802_429_same_target_retry/000_research.md b/devlog/_plan/260802_429_same_target_retry/000_research.md new file mode 100644 index 000000000..2e5dd6673 --- /dev/null +++ b/devlog/_plan/260802_429_same_target_retry/000_research.md @@ -0,0 +1,39 @@ +# 000 — same-target 429 retry: why the client cannot do it + +## 요약 + +Codex turns die instantly when an upstream provider (e.g. BLSC with a single API key) +returns HTTP 429. The proxy forwards the error with `Retry-After` (#514), but Codex +never acts on it. + +## Evidence + +- **Upstream [openai/codex#30471](https://github.com/openai/codex/issues/30471) (open):** + `codex-rs/codex-api/src/provider.rs` has a retry policy with an explicit `retry_429` flag, + and the endpoint configs set it to `false`; HTTP 429 is only mapped to `UsageLimitReached` + when the body matches the Codex usage-limit shape, otherwise it falls through to a generic + transport/API error and surfaces as the misleading "exceeded retry limit" message. The + suggested fix is to reword the error, deliberately preserving `retry_429=false` — there is + no user-facing knob to enable client-side 429 retry. +- **opencodex issue #487 (auto-closed as `not_planned` for missing detail):** the author + requested exactly this feature and noted "The Codex client itself does not retry 429 + (upstream: openai/codex#30471) — it only retries 5xx — so a proxy-side retry would fill a + real gap." +- **opencodex PR #514 (merged):** attaches a client `Retry-After` (upstream header → message + hint → default 2) across Responses, chat completions, Claude messages, and passthrough + paths. Its own test plan lists the Codex recovery check as *unverified*. Claude Code honors + `Retry-After` and absorbs 429s (#507); Codex does not. + +## Current proxy behavior (v2.8.0 / dev) + +- `src/server/responses/core.ts` recovery loop: the only 429 retry is multi-key failover + (`hasKeyPoolFailover` requires ≥2 keys in `apiKeyPool`). Single-key pools no-op, and the + 429 falls through to `rate_limit_error` with `Retry-After`. +- `src/server/chat-completions.ts` and the routed Claude path reuse `handleResponses`, so one + insertion point covers all three inbound surfaces. + +## Conclusion + +Client-side 429 retry does not exist and is not planned. The fix must live in the proxy: +an opt-in wait-and-retry that replays the identical pre-stream request on the same key before +any failover. diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md new file mode 100644 index 000000000..680da3973 --- /dev/null +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -0,0 +1,53 @@ +# 010 — design: `retryOn429` same-target wait-and-retry + +의존: `000_research.md` + +## Goal + +Provider-level opt-in knob: on HTTP 429, wait (upstream `Retry-After` or a fixed interval) +and replay the identical request on the same key, up to `attempts` extra times, before the +existing multi-key failover runs. Default off → zero behavior change for existing setups. + +## Surface + +```jsonc +// ~/.opencodex/config.json → providers. +"retryOn429": { + "enabled": true, // object presence also enables; false disables + "attempts": 3, // extra replays after the first 429 (1..20) + "intervalMs": 5000, // fixed wait when no usable Retry-After + "maxIntervalMs": 60000, // cap for any single wait + "respectRetryAfter": true // prefer the upstream Retry-After when parseable +} +``` + +## Implementation + +- `src/types.ts`: `RateLimitRetryPolicy` interface + `OcxProviderConfig.retryOn429`. +- `src/config.ts`: zod schema entry (strip-unknown, so a typo degrades instead of + rejecting the whole config). +- `src/providers/key-failover.ts`: `rateLimitRetryPolicyFor` (normalize/default) and + `rateLimitRetryDelayMs` (Retry-After seconds/HTTP-date → capped, else `intervalMs`), + reusing the existing `parseRetryAfterMs` cooldown parser. +- `src/usage/log.ts`: new `AttemptRecoveryKind` member `"rate-limit-429"`. +- `src/server/responses/core.ts`: in the pre-stream recovery loop, BEFORE the multi-key + failover `while`, wait then `rebuildAndRefetch("rate-limit-429")`. Abort during the wait + cancels the client request. After attempts are exhausted the existing failover and error + mapping run unchanged. Covers Responses, chat completions, and routed Claude messages + (they all enter `handleResponses`). + +## Safety + +- Pre-stream only: a 429 arrives before any bytes are relayed, so replaying the string-body + request is lossless (same invariant as the transient-5xx layer in `lib/upstream-retry.ts`). +- Ordering: same-key retries run before failover, so "primary-first" users keep their key on + rate-limit blips; failover still works after retries exhaust. +- Latency bound: `attempts × intervalMs` (default 3 × 5 s = 15 s max added latency). +- Final 429 still carries `Retry-After` for clients that honor it (Claude Code). + +## Tests + +- `tests/rate-limit-retry.test.ts` — policy normalization + delay computation (deterministic). +- `tests/server-rate-limit-retry-e2e.test.ts` — single-key replay to success, immediate + passthrough without the knob, exhausted attempts surface 429, and retry-before-failover + ordering with a 2-key pool. diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.md index 0050fa718..8a95c78d6 100644 --- a/docs-site/src/content/docs/reference/configuration.md +++ b/docs-site/src/content/docs/reference/configuration.md @@ -350,6 +350,7 @@ or bind the forward explicitly to loopback (`ssh -L 127.0.0.1:20100:localhost:10 | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Provider-local, disabled-by-default passthrough SSE repair for exact `message` / `reasoning` placeholder ids and missing terminal ids. Downstream only; function-call ids and `call_id` are never rewritten. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced/named choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models that require prior assistant `reasoning_content` to remain in chat history. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Opt-in same-target 429 retry: wait (upstream `Retry-After` or the fixed interval) and replay the identical request on the same key before any key failover. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`, `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | Chat models using a vendor `thinking.enabled` toggle instead of an effort ladder. | | `thinkingBudgetModels?` | `string[]` | Chat models using an integer `thinking_budget`; effort is mapped to a budget fraction. | | `noVisionModels?` | `string[]` | Text-only models — the [vision sidecar](/guides/sidecars/) describes images for them. Matching tolerates an Ollama `:size` tag. | diff --git a/src/config.ts b/src/config.ts index d3927d83b..fc69bb86d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -483,6 +483,13 @@ const providerConfigSchema = z.object({ responsesPath: z.string().min(1).optional(), statelessResponses: z.boolean().optional(), allowPrivateNetwork: z.boolean().optional(), + retryOn429: z.object({ + enabled: z.boolean().optional(), + attempts: z.number().int().min(1).max(20).optional(), + intervalMs: z.number().int().min(100).max(600_000).optional(), + maxIntervalMs: z.number().int().min(100).max(3_600_000).optional(), + respectRetryAfter: z.boolean().optional(), + }).optional(), codexAccountMode: z.enum(["pool", "direct"]).optional(), responsesItemIdRepair: z.object({ message: z.array(z.string().min(1)).optional(), diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 32a308514..5fb4c126e 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -9,7 +9,7 @@ * Modelled after src/codex/routing.ts cooldown logic but scoped to plain API-key pools. */ import { saveConfigPreservingClaudeCode } from "../config"; -import type { OcxConfig, OcxProviderConfig } from "../types"; +import type { OcxConfig, OcxProviderConfig, RateLimitRetryPolicy } from "../types"; import { resolveProviderTransport, type OcxProviderTransport } from "./xai-transport"; import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; @@ -22,6 +22,14 @@ interface KeyCooldown { const DEFAULT_COOLDOWN_MS = 60_000; const MAX_COOLDOWN_MS = 10 * 60_000; // cap at 10 min for api-key rotation +const DEFAULT_RATE_LIMIT_RETRY = { + enabled: true, + attempts: 3, + intervalMs: 5_000, + maxIntervalMs: 60_000, + respectRetryAfter: true, +} as const satisfies Required; + /** Map<`${providerName}\0${keyId}`, KeyCooldown> */ const keyCooldowns = new Map(); @@ -65,6 +73,42 @@ export function hasKeyPoolFailover(provider: OcxProviderConfig): boolean { return (provider.apiKeyPool?.length ?? 0) >= 2; } +/** + * Normalize a provider's `retryOn429` policy, or return null when the knob is absent or + * explicitly disabled. The returned policy is fully defaulted so callers never re-check fields. + */ +export function rateLimitRetryPolicyFor( + provider: Pick, +): Required | null { + const policy = provider.retryOn429; + if (!policy || policy.enabled === false) return null; + return { + enabled: policy.enabled ?? DEFAULT_RATE_LIMIT_RETRY.enabled, + attempts: policy.attempts ?? DEFAULT_RATE_LIMIT_RETRY.attempts, + intervalMs: policy.intervalMs ?? DEFAULT_RATE_LIMIT_RETRY.intervalMs, + maxIntervalMs: policy.maxIntervalMs ?? DEFAULT_RATE_LIMIT_RETRY.maxIntervalMs, + respectRetryAfter: policy.respectRetryAfter ?? DEFAULT_RATE_LIMIT_RETRY.respectRetryAfter, + }; +} + +/** + * Wait before the next same-target replay: upstream Retry-After (seconds or HTTP-date) when + * `respectRetryAfter` is on and the header parses, capped at `maxIntervalMs`; otherwise the + * fixed `intervalMs`. Malformed headers fall back to the fixed interval. + */ +export function rateLimitRetryDelayMs( + policy: Required, + retryAfterHeader: string | null | undefined, + now = Date.now(), +): number { + const raw = retryAfterHeader?.trim(); + if (policy.respectRetryAfter && raw) { + const parsed = parseRetryAfterMs(raw, now); + if (parsed !== undefined) return Math.min(parsed, policy.maxIntervalMs); + } + return policy.intervalMs; +} + /** * Record a 429 for the current key and attempt to switch to the next available one. * diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index fabbc6ad8..5ba402090 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -85,7 +85,13 @@ import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; -import { fetchWithResetRetry, fetchWithTransientRetry, applyUpstreamRecoveryInit } from "../../lib/upstream-retry"; +import { + applyUpstreamRecoveryInit, + cancelResponseBodyBestEffort, + fetchWithResetRetry, + fetchWithTransientRetry, + sleepWithAbort, +} from "../../lib/upstream-retry"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { createTranslatorBudget, isTranslatorBudgetExceededError, type TranslatorBudget } from "../../lib/translator-budget"; import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; @@ -96,7 +102,12 @@ import { isUsageDebugEnabled } from "../../usage/debug"; import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress"; import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve"; import type { InboundWire } from "../../providers/registry"; -import { hasKeyPoolFailover, rotateProviderTransportOn429 } from "../../providers/key-failover"; +import { + hasKeyPoolFailover, + rateLimitRetryDelayMs, + rateLimitRetryPolicyFor, + rotateProviderTransportOn429, +} from "../../providers/key-failover"; import { shouldAttemptImageTierRetry } from "../image-retry"; import { resolveProviderTransport } from "../../providers/xai-transport"; import type { WsData } from "../ws-bridge"; @@ -2365,6 +2376,36 @@ async function handleResponsesInner( continue recovery; } + // Same-target 429 wait-and-retry (opt-in `retryOn429`, issue #487). Codex never retries + // 429 itself (it retries 5xx only), and single-key pools cannot use the failover below, + // so wait (Retry-After or the fixed interval) and replay the IDENTICAL request on the + // same key first. Pre-stream only: a 429 arrives before any bytes are relayed, so the + // replay is lossless. Runs before key failover so "primary-first" setups keep the same + // key on rate-limit blips; only after the attempts are exhausted does failover run. + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; + while ( + upstreamResponse.status === 429 + && rateLimitPolicy !== null + && rateLimitRetries < rateLimitPolicy.attempts + ) { + rateLimitRetries += 1; + try { + await sleepWithAbort( + rateLimitRetryDelayMs(rateLimitPolicy, upstreamResponse.headers.get("retry-after"), Date.now()), + options.abortSignal, + ); + } catch { + cleanupUpstreamAbort(); + upstream.abort(); + return clientCancelledResponse(); + } + cancelResponseBodyBestEffort(upstreamResponse); + const result = await rebuildAndRefetch("rate-limit-429"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + } + // Multi-key 429 failover: rotate to the next pool key (cooldown-aware) and retry the // SAME request once per remaining key. OAuth/forward providers and single-key pools // return null immediately, so this stays a no-op for them (src/providers/key-failover.ts). diff --git a/src/types.ts b/src/types.ts index bae193fb6..d478e153b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -905,6 +905,25 @@ export interface ResponsesItemIdRepairConfig { repairMissingTerminalIds?: boolean; } +/** + * Same-target 429 wait-and-retry policy (`providers..retryOn429`). When present and not + * explicitly disabled, the proxy waits and replays the identical request on the same key before + * any key failover. All fields optional; the runtime applies defaults (attempts=3, + * intervalMs=5000, maxIntervalMs=60000, respectRetryAfter=true, enabled=true). + */ +export interface RateLimitRetryPolicy { + /** Master switch. The presence of the object also enables the policy (default true). */ + enabled?: boolean; + /** Extra replay attempts after the first 429 (1..20, default 3). */ + attempts?: number; + /** Fixed wait between attempts when the upstream sends no usable Retry-After (default 5000). */ + intervalMs?: number; + /** Cap for any single wait, including an upstream Retry-After (default 60000). */ + maxIntervalMs?: number; + /** Prefer the upstream Retry-After header when present and parseable (default true). */ + respectRetryAfter?: boolean; +} + export interface OcxProviderConfig { adapter: string; /** Cursor MCP compatibility bounds; positive integers when configured. */ @@ -1083,6 +1102,13 @@ export interface OcxProviderConfig { autoToolChoiceOnlyModels?: string[]; /** Model ids that expect prior assistant `reasoning_content` to be preserved in chat history. */ preserveReasoningContentModels?: string[]; + /** + * Opt-in same-target 429 retry policy. Codex itself never retries 429 (it retries 5xx only, + * openai/codex#30471), and single-key pools have no failover, so the proxy waits and replays + * the identical request on the same key before any failover. Pre-stream only: a 429 arrives + * before any response bytes are relayed, so the replay is lossless. + */ + retryOn429?: RateLimitRetryPolicy; /** * Model ids whose OpenAI-compatible chat endpoint accepts `reasoning_split: true` and returns * thinking separately in `reasoning_content` / `reasoning_details` instead of visible content. diff --git a/src/usage/log.ts b/src/usage/log.ts index 3c588c758..df7346e6b 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -12,6 +12,7 @@ export type AttemptRecoveryKind = | "connection-reset" | "oauth-401" | "key-429" + | "rate-limit-429" | "anthropic-oauth-429" | "image-413"; diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index 5a39c59e2..2fd560765 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -219,6 +219,14 @@ ordinary 5xx errors are not replayed. Completion fallback rebuilds only replayab the original user/tool-result turn for reasoning-only attempts, supplies neutral non-empty carriers for empty tool output, and validates role alternation plus tool-use/result pairing before transport. +Provider-level `retryOn429` (devlog 260802_429_same_target_retry) is the generic, opt-in +same-target 429 retry for key-auth providers, primarily single-key pools that cannot use +multi-key failover. In the pre-stream recovery loop, a 429 waits (`Retry-After` or the fixed +interval, capped) and replays the identical request on the same key before any failover, up to +`attempts` extra times. Codex never retries 429 client-side (openai/codex#30471), so this is the +only defense for those providers; the final 429 still carries `Retry-After` for clients that +honor it. + [Decision Log] - 목적과 의도: Prevent Kiro progress from becoming a false final answer, reject invalid empty completion retries, and stop concurrent transient 429s from consuming independent retry budgets. - 기존 구현 및 제약 조건: Kiro text has no trustworthy phase; stop metadata arrives only at stream end; the private completion tool is adapter-owned; normal parallel tool traffic must remain parallel; client cancellation must interrupt all waits. diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts new file mode 100644 index 000000000..537a6ae56 --- /dev/null +++ b/tests/rate-limit-retry.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { + rateLimitRetryDelayMs, + rateLimitRetryPolicyFor, +} from "../src/providers/key-failover"; +import type { OcxProviderConfig } from "../src/types"; + +describe("rateLimitRetryPolicyFor", () => { + test("null when absent or explicitly disabled", () => { + expect(rateLimitRetryPolicyFor({} as OcxProviderConfig)).toBeNull(); + expect(rateLimitRetryPolicyFor({ retryOn429: { enabled: false } } as OcxProviderConfig)).toBeNull(); + }); + + test("applies defaults when the object is present", () => { + expect(rateLimitRetryPolicyFor({ retryOn429: {} } as OcxProviderConfig)).toEqual({ + enabled: true, + attempts: 3, + intervalMs: 5_000, + maxIntervalMs: 60_000, + respectRetryAfter: true, + }); + }); + + test("honors explicit values", () => { + expect(rateLimitRetryPolicyFor({ + retryOn429: { attempts: 10, intervalMs: 1_000, maxIntervalMs: 5_000, respectRetryAfter: false }, + } as OcxProviderConfig)).toEqual({ + enabled: true, + attempts: 10, + intervalMs: 1_000, + maxIntervalMs: 5_000, + respectRetryAfter: false, + }); + }); +}); + +describe("rateLimitRetryDelayMs", () => { + const policy = rateLimitRetryPolicyFor({ retryOn429: {} } as OcxProviderConfig)!; + + test("fixed interval when no header is present", () => { + expect(rateLimitRetryDelayMs(policy, null, 1_000_000)).toBe(5_000); + expect(rateLimitRetryDelayMs(policy, undefined, 1_000_000)).toBe(5_000); + }); + + test("honors Retry-After seconds and caps it at maxIntervalMs", () => { + expect(rateLimitRetryDelayMs(policy, "2", 1_000_000)).toBe(2_000); + expect(rateLimitRetryDelayMs(policy, "3600", 1_000_000)).toBe(60_000); + }); + + test("malformed Retry-After falls back to the fixed interval", () => { + expect(rateLimitRetryDelayMs(policy, "soon", 1_000_000)).toBe(5_000); + }); + + test("respectRetryAfter=false ignores the header", () => { + const p = rateLimitRetryPolicyFor({ + retryOn429: { respectRetryAfter: false, intervalMs: 111 }, + } as OcxProviderConfig)!; + expect(rateLimitRetryDelayMs(p, "2", 1_000_000)).toBe(111); + }); +}); diff --git a/tests/server-rate-limit-retry-e2e.test.ts b/tests/server-rate-limit-retry-e2e.test.ts new file mode 100644 index 000000000..414a7ecca --- /dev/null +++ b/tests/server-rate-limit-retry-e2e.test.ts @@ -0,0 +1,236 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../src/config"; +import { clearKeyCooldowns } from "../src/providers/key-failover"; +import { startServer } from "../src/server"; +import type { OcxConfig } from "../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-ratelimit-e2e-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-ratelimit-e2e-")); + process.env.OPENCODEX_HOME = testDir; + clearKeyCooldowns(); +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) rmSync(testDir, { recursive: true, force: true }); + clearKeyCooldowns(); +}); + +const okChatCompletion = JSON.stringify({ + id: "chatcmpl-ratelimit", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "ok after retry" }, finish_reason: "stop" }], + usage: { prompt_tokens: 3, completion_tokens: 2, total_tokens: 5 }, +}); + +async function postResponses(serverUrl: string, model: string): Promise { + return fetch(new URL("/v1/responses", serverUrl), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, input: "hello", stream: false }), + }); +} + +describe("server same-target 429 retry (end-to-end)", () => { + test("single-key provider replays the identical request until upstream succeeds", async () => { + const originalFetch = globalThis.fetch; + const seenBodies: string[] = []; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + seenBodies.push(String(init?.body)); + if (seenBodies.length <= 2) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "retry-after": "1", "content-type": "application/json" }, + }); + } + return new Response(okChatCompletion, { headers: { "content-type": "application/json" } }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + retryOn429: { attempts: 2, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "blsc/DeepSeek-V4-Flash"); + expect(res.status).toBe(200); + const json = await res.json() as { output?: { type: string; content?: { text?: string }[] }[] }; + expect(json.output?.find(o => o.type === "message")?.content?.[0]?.text).toBe("ok after retry"); + expect(seenBodies).toHaveLength(3); + expect(seenBodies[0]).toBe(seenBodies[1]); + expect(seenBodies[1]).toBe(seenBodies[2]); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); + + test("without retryOn429 the 429 surfaces immediately with Retry-After", async () => { + const originalFetch = globalThis.fetch; + let sends = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + sends += 1; + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "retry-after": "17", "content-type": "application/json" }, + }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "blsc/DeepSeek-V4-Flash"); + expect(res.status).toBe(429); + expect(res.headers.get("retry-after")).toBe("17"); + const json = await res.json() as { error?: { type?: string } }; + expect(json.error?.type).toBe("rate_limit_error"); + expect(sends).toBe(1); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); + + test("exhausted attempts surface the 429", async () => { + const originalFetch = globalThis.fetch; + let sends = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + sends += 1; + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "retry-after": "1", "content-type": "application/json" }, + }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "blsc/DeepSeek-V4-Flash"); + expect(res.status).toBe(429); + expect(sends).toBe(2); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); + + test("same-key retries run before multi-key failover, which still works after they exhaust", async () => { + const originalFetch = globalThis.fetch; + const seenAuth: string[] = []; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + const auth = new Headers(init?.headers).get("authorization") ?? ""; + seenAuth.push(auth); + if (auth.includes("key-beta")) { + return new Response(okChatCompletion, { headers: { "content-type": "application/json" } }); + } + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "retry-after": "1", "content-type": "application/json" }, + }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + apiKeyPool: [ + { id: "k1", key: "key-alpha-000111222333", addedAt: 1 }, + { id: "k2", key: "key-beta-444555666777", addedAt: 2 }, + ], + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "blsc/DeepSeek-V4-Flash"); + expect(res.status).toBe(200); + expect(seenAuth).toEqual([ + "Bearer key-alpha-000111222333", + "Bearer key-alpha-000111222333", + "Bearer key-beta-444555666777", + ]); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); +}); From 2efb887b82202213f624439b18bd4abf85d01a68 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 02:22:31 +0800 Subject: [PATCH 02/29] fix(proxy): address audit round-1 findings for retryOn429 - usage/log: whitelist the rate-limit-429 recovery kind so persisted usage rows and post-restart /api/logs keep the reason - key-failover: gate retryOn429 to key-auth providers (no same-token OAuth replays, no silent no-op on forward passthrough); accept Retry-After 0 as immediate - config: cap maxIntervalMs at the effective 10-minute cooldown ceiling - core: hoist the retry budget outside the recovery loop so 413/401 replays cannot re-arm it; cancel the unread 429 body before aborting on client cancel - gui: add rate-limit-429 (and pre-existing anthropic-oauth-429) to the Logs union - tests: OAuth/forward gating, HTTP-date Retry-After, Retry-After 0, persisted recovery-kind, and a deterministic direct-handler abort test (real-socket disconnect propagation is async in Bun, so the e2e version was timing-flaky) - docs: key-auth-only note, maxIntervalMs cap, abort-propagation nuance, locale rows --- .../010_design.md | 34 +++++-- .../docs/ja/reference/configuration.md | 1 + .../docs/ko/reference/configuration.md | 1 + .../content/docs/reference/configuration.md | 2 +- .../docs/ru/reference/configuration.md | 1 + .../docs/zh-cn/reference/configuration.md | 1 + gui/src/pages/Logs.tsx | 2 + src/config.ts | 4 +- src/providers/key-failover.ts | 11 ++- src/server/responses/core.ts | 7 +- src/usage/log.ts | 1 + structure/04_transports-and-sidecars.md | 13 ++- tests/rate-limit-retry.test.ts | 91 ++++++++++++++++++- tests/usage-log.test.ts | 24 +++++ 14 files changed, 172 insertions(+), 21 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index 680da3973..74f7c1a6e 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -24,17 +24,20 @@ existing multi-key failover runs. Default off → zero behavior change for exist ## Implementation - `src/types.ts`: `RateLimitRetryPolicy` interface + `OcxProviderConfig.retryOn429`. -- `src/config.ts`: zod schema entry (strip-unknown, so a typo degrades instead of - rejecting the whole config). +- `src/config.ts`: zod schema entry (zod's default strip inside the object — an unknown key is + dropped, never a config-rejecting error; the outer provider schema stays passthrough). - `src/providers/key-failover.ts`: `rateLimitRetryPolicyFor` (normalize/default) and `rateLimitRetryDelayMs` (Retry-After seconds/HTTP-date → capped, else `intervalMs`), - reusing the existing `parseRetryAfterMs` cooldown parser. + reusing the existing `parseRetryAfterMs` cooldown parser. Key-auth providers only: OAuth and + forward credentials are never replayed on the same token. - `src/usage/log.ts`: new `AttemptRecoveryKind` member `"rate-limit-429"`. - `src/server/responses/core.ts`: in the pre-stream recovery loop, BEFORE the multi-key failover `while`, wait then `rebuildAndRefetch("rate-limit-429")`. Abort during the wait - cancels the client request. After attempts are exhausted the existing failover and error - mapping run unchanged. Covers Responses, chat completions, and routed Claude messages - (they all enter `handleResponses`). + cancels the client request (the unread 429 body is released first). The retry budget lives + OUTSIDE the recovery loop, so a 413/401 replay that comes back 429 cannot re-arm a fresh + budget — bounded to `attempts` per request. After attempts are exhausted the existing + failover and error mapping run unchanged. Covers Responses, chat completions, and routed + Claude messages (they all enter `handleResponses`). ## Safety @@ -42,12 +45,27 @@ existing multi-key failover runs. Default off → zero behavior change for exist request is lossless (same invariant as the transient-5xx layer in `lib/upstream-retry.ts`). - Ordering: same-key retries run before failover, so "primary-first" users keep their key on rate-limit blips; failover still works after retries exhaust. -- Latency bound: `attempts × intervalMs` (default 3 × 5 s = 15 s max added latency). +- Latency bound: worst case `attempts × maxIntervalMs` (default 3 × 60 s = 180 s) when + honoring upstream `Retry-After`; `attempts × intervalMs` (default 15 s) when + `respectRetryAfter=false` or no header is present. +- Abort during the wait: the sleep is abort-aware — when the server observes the client + disconnect (Bun propagates this asynchronously, observed 1–10 s), the wait is interrupted, + the unread 429 body is released, and the request is cancelled with 499 before any replay. + Because the propagation is async, a replay can still precede the cancel if the interval + elapses first; that is bounded by the same `attempts` budget. +- Concurrency: each request honors its own policy independently — no process-wide cooldown is + shared between concurrent requests (unlike the Kiro 429 pattern). Opt-in and bounded, so a + storm multiplies upstream volume by at most `attempts + 1` per request. - Final 429 still carries `Retry-After` for clients that honor it (Claude Code). ## Tests -- `tests/rate-limit-retry.test.ts` — policy normalization + delay computation (deterministic). +- `tests/rate-limit-retry.test.ts` — policy normalization (incl. OAuth/forward gating) + + delay computation (seconds, HTTP-date, `0`, malformed, cap; deterministic) + abort during + the wait: a directly-invoked `handleResponses` with a controlled abort signal returns 499, + cancels the unread 429 body, and performs no further upstream sends (deterministic — no + real-socket disconnect timing involved). +- `tests/usage-log.test.ts` — the `rate-limit-429` recovery kind survives persisted usage logs. - `tests/server-rate-limit-retry-e2e.test.ts` — single-key replay to success, immediate passthrough without the knob, exhausted attempts surface 429, and retry-before-failover ordering with a 2-key pool. diff --git a/docs-site/src/content/docs/ja/reference/configuration.md b/docs-site/src/content/docs/ja/reference/configuration.md index ed1c42511..bc975569a 100644 --- a/docs-site/src/content/docs/ja/reference/configuration.md +++ b/docs-site/src/content/docs/ja/reference/configuration.md @@ -194,6 +194,7 @@ timing side channel を防ぐため定数時間(`timingSafeEqual`)で比較 | `parallelToolCalls?` | `boolean` | 並列ツール呼び出しをオン/オフします。OpenAI Chat はデフォルト on で、chat 以外のアダプターは明示的な `true` でのみサポートを公表します。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` で `auto` または `none` だけを受け付けるモデル。強制/指定選択は downgrade します。 | | `preserveReasoningContentModels?` | `string[]` | 前の assistant `reasoning_content` を chat history に維持すべきモデル。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | key-auth プロバイダーのみ。オプトインの同一ターゲット 429 リトライ: 待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は 600000 で上限)、`respectRetryAfter: true`。 | | `thinkingToggleModels?` | `string[]` | effort 段階の代わりに vendor `thinking.enabled` toggle を使う chat モデル。 | | `thinkingBudgetModels?` | `string[]` | 整数 `thinking_budget` を使う chat モデル。effort を budget 比率にマッピングします。 | | `noVisionModels?` | `string[]` | テキスト専用モデル。[ビジョンサイドカー](/ja/guides/sidecars/) が画像を説明します。Ollama の `:size` タグも一致させます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration.md b/docs-site/src/content/docs/ko/reference/configuration.md index cb947d866..42529163c 100644 --- a/docs-site/src/content/docs/ko/reference/configuration.md +++ b/docs-site/src/content/docs/ko/reference/configuration.md @@ -204,6 +204,7 @@ timing side channel을 막기 위해 상수 시간(`timingSafeEqual`)으로 비 | `parallelToolCalls?` | `boolean` | 병렬 툴 호출을 켜거나 끕니다. OpenAI Chat은 기본 on이며, chat 외 어댑터는 명시적인 `true`에서만 지원을 알립니다. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`에서 `auto` 또는 `none`만 받는 모델. 강제/지정 선택은 downgrade합니다. | | `preserveReasoningContentModels?` | `string[]` | 이전 assistant `reasoning_content`를 chat history에 유지해야 하는 모델. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | key-auth 프로바이더 전용. 동일 대상 429 재시도: 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 600000으로 상한), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | effort 단계 대신 vendor `thinking.enabled` toggle을 쓰는 chat 모델. | | `thinkingBudgetModels?` | `string[]` | 정수 `thinking_budget`을 쓰는 chat 모델. effort를 budget 비율로 매핑합니다. | | `noVisionModels?` | `string[]` | 텍스트 전용 모델. [비전 사이드카](/ko/guides/sidecars/)가 이미지를 설명합니다. Ollama의 `:size` 태그도 일치시킵니다. | diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.md index 8a95c78d6..ac48b610f 100644 --- a/docs-site/src/content/docs/reference/configuration.md +++ b/docs-site/src/content/docs/reference/configuration.md @@ -350,7 +350,7 @@ or bind the forward explicitly to loopback (`ssh -L 127.0.0.1:20100:localhost:10 | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Provider-local, disabled-by-default passthrough SSE repair for exact `message` / `reasoning` placeholder ids and missing terminal ids. Downstream only; function-call ids and `call_id` are never rewritten. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced/named choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models that require prior assistant `reasoning_content` to remain in chat history. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Opt-in same-target 429 retry: wait (upstream `Retry-After` or the fixed interval) and replay the identical request on the same key before any key failover. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`, `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Key-auth providers only. Opt-in same-target 429 retry: wait (upstream `Retry-After` or the fixed interval) and replay the identical request on the same key before any key failover. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (a single wait is capped at 600000), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | Chat models using a vendor `thinking.enabled` toggle instead of an effort ladder. | | `thinkingBudgetModels?` | `string[]` | Chat models using an integer `thinking_budget`; effort is mapped to a budget fraction. | | `noVisionModels?` | `string[]` | Text-only models — the [vision sidecar](/guides/sidecars/) describes images for them. Matching tolerates an Ollama `:size` tag. | diff --git a/docs-site/src/content/docs/ru/reference/configuration.md b/docs-site/src/content/docs/ru/reference/configuration.md index 93e58f6df..80a3c01b5 100644 --- a/docs-site/src/content/docs/ru/reference/configuration.md +++ b/docs-site/src/content/docs/ru/reference/configuration.md @@ -228,6 +228,7 @@ Responses и Chat Completions принимают только выделенны | `parallelToolCalls?` | `boolean` | Включает/отключает параллельные вызовы инструментов. В OpenAI Chat включено по умолчанию; не-chat-адаптеры объявляют поддержку только при явном `true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; принудительные/именованные выборы понижаются. | | `preserveReasoningContentModels?` | `string[]` | Модели, требующие сохранять предыдущий assistant `reasoning_content` в истории чата. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с key-auth. Опциональный повтор при 429 на том же таргете: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (одно ожидание ограничено 600000), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие вендорский переключатель `thinking.enabled` вместо лестницы уровней рассуждений. | | `thinkingBudgetModels?` | `string[]` | Chat-модели, использующие целочисленный `thinking_budget`; уровень отображается в долю бюджета. | | `noVisionModels?` | `string[]` | Модели только для текста — [vision-сайдкар](/ru/guides/sidecars/) описывает для них изображения. Сопоставление допускает тег Ollama `:size`. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration.md b/docs-site/src/content/docs/zh-cn/reference/configuration.md index 17c2249bb..5ec692b55 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration.md @@ -196,6 +196,7 @@ Codex Direct 透传,两个 bearer 域不能混淆。仪表盘的 API 标签页 | `parallelToolCalls?` | `boolean` | 启用或禁用并行工具调用。OpenAI Chat 默认开启;非 chat adapter 只有显式为 `true` 时才公布支持。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型;forced/named 选择会降级。 | | `preserveReasoningContentModels?` | `string[]` | 要求在 chat history 中保留先前 assistant `reasoning_content` 的模型。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 key-auth 提供商。可选的同目标 429 重试:等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待上限 600000)、`respectRetryAfter: true`。 | | `thinkingToggleModels?` | `string[]` | 使用 vendor `thinking.enabled` toggle,而不是 effort ladder 的 chat 模型。 | | `thinkingBudgetModels?` | `string[]` | 使用整数 `thinking_budget` 的 chat 模型;effort 会映射成 budget 比例。 | | `noVisionModels?` | `string[]` | 纯文本模型;[视觉 sidecar](/zh-cn/guides/sidecars/) 会为它们描述图像。匹配时容忍 Ollama `:size` 标签。 | diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 56c7d80ba..fb843567a 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -81,6 +81,8 @@ type AttemptRecoveryKind = | "connection-reset" | "oauth-401" | "key-429" + | "rate-limit-429" + | "anthropic-oauth-429" | "image-413"; interface LogAttempt { diff --git a/src/config.ts b/src/config.ts index fc69bb86d..3474325ec 100644 --- a/src/config.ts +++ b/src/config.ts @@ -487,7 +487,9 @@ const providerConfigSchema = z.object({ enabled: z.boolean().optional(), attempts: z.number().int().min(1).max(20).optional(), intervalMs: z.number().int().min(100).max(600_000).optional(), - maxIntervalMs: z.number().int().min(100).max(3_600_000).optional(), + // The effective cap for a single wait is MAX_COOLDOWN_MS (10 min) in key-failover.ts; + // larger configured values would be dead config. + maxIntervalMs: z.number().int().min(100).max(600_000).optional(), respectRetryAfter: z.boolean().optional(), }).optional(), codexAccountMode: z.enum(["pool", "direct"]).optional(), diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 5fb4c126e..139807f9b 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -42,7 +42,7 @@ function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): if (!text) return undefined; if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); - if (Number.isFinite(seconds) && seconds > 0) { + if (Number.isFinite(seconds) && seconds >= 0) { return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS); } } @@ -74,14 +74,17 @@ export function hasKeyPoolFailover(provider: OcxProviderConfig): boolean { } /** - * Normalize a provider's `retryOn429` policy, or return null when the knob is absent or - * explicitly disabled. The returned policy is fully defaulted so callers never re-check fields. + * Normalize a provider's `retryOn429` policy, or return null when the knob is absent, + * explicitly disabled, or the provider is not key-auth (OAuth/forward credentials must not be + * replayed on the same token, and forward passthrough never reaches the recovery loop anyway). + * The returned policy is fully defaulted so callers never re-check fields. */ export function rateLimitRetryPolicyFor( - provider: Pick, + provider: Pick, ): Required | null { const policy = provider.retryOn429; if (!policy || policy.enabled === false) return null; + if (provider.authMode === "oauth" || provider.authMode === "forward") return null; return { enabled: policy.enabled ?? DEFAULT_RATE_LIMIT_RETRY.enabled, attempts: policy.attempts ?? DEFAULT_RATE_LIMIT_RETRY.attempts, diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 5ba402090..7bbeb622c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2303,6 +2303,10 @@ async function handleResponsesInner( let imageTierBias = 0; let imageRetryAttempted = false; let oauth401ReplayAttempted = false; + // Same-target 429 retry budget lives OUTSIDE the recovery loop so a 413/401 replay that + // comes back 429 cannot silently re-arm a fresh budget (bounded to `attempts` per request). + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; const rebuildAndRefetch = async ( recovery: AttemptRecoveryKind, ): Promise => { @@ -2382,8 +2386,6 @@ async function handleResponsesInner( // same key first. Pre-stream only: a 429 arrives before any bytes are relayed, so the // replay is lossless. Runs before key failover so "primary-first" setups keep the same // key on rate-limit blips; only after the attempts are exhausted does failover run. - const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); - let rateLimitRetries = 0; while ( upstreamResponse.status === 429 && rateLimitPolicy !== null @@ -2396,6 +2398,7 @@ async function handleResponsesInner( options.abortSignal, ); } catch { + cancelResponseBodyBestEffort(upstreamResponse); cleanupUpstreamAbort(); upstream.abort(); return clientCancelledResponse(); diff --git a/src/usage/log.ts b/src/usage/log.ts index df7346e6b..8861ec67f 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -174,6 +174,7 @@ const ATTEMPT_RECOVERY_KINDS = new Set([ "connection-reset", "oauth-401", "key-429", + "rate-limit-429", "anthropic-oauth-429", "image-413", ]); diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index 2fd560765..359de4615 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -223,9 +223,16 @@ Provider-level `retryOn429` (devlog 260802_429_same_target_retry) is the generic same-target 429 retry for key-auth providers, primarily single-key pools that cannot use multi-key failover. In the pre-stream recovery loop, a 429 waits (`Retry-After` or the fixed interval, capped) and replays the identical request on the same key before any failover, up to -`attempts` extra times. Codex never retries 429 client-side (openai/codex#30471), so this is the -only defense for those providers; the final 429 still carries `Retry-After` for clients that -honor it. +`attempts` extra times per request (the budget lives outside the recovery loop, so a 413/401 +replay cannot re-arm it). Codex never retries 429 client-side (openai/codex#30471), so this is +the only defense for those providers; the final 429 still carries `Retry-After` for clients that +honor it. Concurrent requests each honor their own policy — there is no process-wide shared +cooldown (unlike the Kiro pattern), so a rate-limit storm multiplies upstream volume by at most +`attempts + 1` per request. The wait is abort-aware: once the server observes the client +disconnect (Bun propagates it asynchronously, observed 1–10 s), the sleep is interrupted, the +unread 429 body is released, and the request is cancelled with 499 before any replay; because +the propagation is async, a replay may precede the cancel if the interval elapses first +(bounded by the same `attempts` budget). [Decision Log] - 목적과 의도: Prevent Kiro progress from becoming a false final answer, reject invalid empty completion retries, and stop concurrent transient 429s from consuming independent retry budgets. diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts index 537a6ae56..f86f8e4e8 100644 --- a/tests/rate-limit-retry.test.ts +++ b/tests/rate-limit-retry.test.ts @@ -1,9 +1,10 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { rateLimitRetryDelayMs, rateLimitRetryPolicyFor, } from "../src/providers/key-failover"; -import type { OcxProviderConfig } from "../src/types"; +import { handleResponses } from "../src/server/responses"; +import type { OcxConfig, OcxProviderConfig } from "../src/types"; describe("rateLimitRetryPolicyFor", () => { test("null when absent or explicitly disabled", () => { @@ -11,6 +12,21 @@ describe("rateLimitRetryPolicyFor", () => { expect(rateLimitRetryPolicyFor({ retryOn429: { enabled: false } } as OcxProviderConfig)).toBeNull(); }); + test("null for OAuth and forward providers (same-token replays are never attempted)", () => { + expect(rateLimitRetryPolicyFor({ + authMode: "oauth", + retryOn429: {}, + } as OcxProviderConfig)).toBeNull(); + expect(rateLimitRetryPolicyFor({ + authMode: "forward", + retryOn429: {}, + } as OcxProviderConfig)).toBeNull(); + expect(rateLimitRetryPolicyFor({ + authMode: "key", + retryOn429: {}, + } as OcxProviderConfig)).not.toBeNull(); + }); + test("applies defaults when the object is present", () => { expect(rateLimitRetryPolicyFor({ retryOn429: {} } as OcxProviderConfig)).toEqual({ enabled: true, @@ -47,6 +63,15 @@ describe("rateLimitRetryDelayMs", () => { expect(rateLimitRetryDelayMs(policy, "3600", 1_000_000)).toBe(60_000); }); + test("honors an HTTP-date Retry-After", () => { + const now = Date.parse("2026-10-21T07:27:30Z"); + expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:28:00 GMT", now)).toBe(30_000); + }); + + test("Retry-After 0 retries immediately instead of falling back to the interval", () => { + expect(rateLimitRetryDelayMs(policy, "0", 1_000_000)).toBe(1); + }); + test("malformed Retry-After falls back to the fixed interval", () => { expect(rateLimitRetryDelayMs(policy, "soon", 1_000_000)).toBe(5_000); }); @@ -58,3 +83,65 @@ describe("rateLimitRetryDelayMs", () => { expect(rateLimitRetryDelayMs(p, "2", 1_000_000)).toBe(111); }); }); + +describe("retry loop client-abort handling", () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test("abort during the wait interrupts the sleep, cancels the 429 body, and returns 499 without replaying", async () => { + let sends = 0; + let upstreamBodyCancelled = false; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + sends += 1; + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(JSON.stringify({ error: { message: "rate limited" } }))); + controller.close(); + }, + cancel() { + upstreamBodyCancelled = true; + }, + }), { status: 429, headers: { "content-type": "application/json" } }); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config = { + port: 0, + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + retryOn429: { attempts: 3, intervalMs: 30_000, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + + const abort = new AbortController(); + const pending = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "blsc/DeepSeek-V4-Flash", input: "hello", stream: false }), + }), config, { model: "blsc/DeepSeek-V4-Flash", provider: "blsc" }, { abortSignal: abort.signal }); + + // Wait until the first upstream 429 lands and the retry sleep is running. + for (let i = 0; i < 100 && sends === 0; i += 1) await Bun.sleep(10); + expect(sends).toBe(1); + + abort.abort(new DOMException("client disconnected", "AbortError")); + const response = await pending; + expect(response.status).toBe(499); + expect(sends).toBe(1); + expect(upstreamBodyCancelled).toBe(true); + const body = await response.json() as { error?: { code?: string } }; + expect(body.error?.code).toBe("client_cancelled"); + }); +}); diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index f9a5b103b..702bdbc1b 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -36,6 +36,30 @@ afterEach(() => { }); describe("usage log", () => { + test("persists the rate-limit-429 recovery kind on attempts", () => { + appendUsageEntry({ + requestId: "ocx-ratelimit-kind", + timestamp: 1, + provider: "blsc", + model: "blsc/DeepSeek-V4-Flash", + status: 429, + durationMs: 4, + usageStatus: "reported", + attempts: [{ + ordinal: 1, + provider: "blsc", + model: "blsc/DeepSeek-V4-Flash", + adapter: "openai-chat", + status: 429, + durationMs: 4, + sendCount: 2, + recoveryKinds: ["rate-limit-429", "rate-limit-429"], + usageStatus: "reported", + }], + } as never); + expect(readUsageEntries()[0]?.attempts?.[0]?.recoveryKinds).toEqual(["rate-limit-429"]); + }); + const persistedLine = (requestId: string) => JSON.stringify({ requestId, timestamp: 1, From a06a416024ea302a48e12e52513c1f9947c33f1d Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 02:46:06 +0800 Subject: [PATCH 03/29] fix(proxy): extend retryOn429 to all key-auth surfaces (audit round-3 xhigh) - core: the Responses passthrough wire (openai-responses key-auth gateways, e.g. the built-in DeepSeek preset) now replays 429 on the same key pre-relay, before the forward-pool logic; Anthropic terminal-guard continuations replay before key/account failover - images/loop + web-search/loop: same-target replay before on429 key rotation via a new retryOn429Policy dep (abort-aware, heartbeat seams preserved) - key-failover: the fixed fallback is now capped at maxIntervalMs (a single wait never exceeds the cap); authMode local runtimes are gated out alongside oauth/forward, so the knob matches the documented API-key scope exactly - tests: key-auth openai-responses passthrough e2e, terminal-guard continuation replay, image/web-search same-key replay (rotation stays zero), fallback cap, local-mode gating - docs: coverage and cap wording in devlog 010, structure/04, and all five configuration locale rows --- .../010_design.md | 24 ++- .../docs/ja/reference/configuration.md | 2 +- .../docs/ko/reference/configuration.md | 2 +- .../content/docs/reference/configuration.md | 2 +- .../docs/ru/reference/configuration.md | 2 +- .../docs/zh-cn/reference/configuration.md | 2 +- src/images/loop.ts | 31 +++- src/providers/key-failover.ts | 12 +- src/server/responses/core.ts | 166 +++++++++++++----- src/web-search/loop.ts | 31 +++- structure/04_transports-and-sidecars.md | 20 ++- tests/images/loop.test.ts | 33 ++++ tests/rate-limit-retry.test.ts | 14 +- tests/server-rate-limit-retry-e2e.test.ts | 59 +++++++ tests/terminal-guard-server.test.ts | 43 +++++ tests/web-search.test.ts | 53 ++++++ 16 files changed, 427 insertions(+), 69 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index 74f7c1a6e..7a0c0542f 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -28,16 +28,25 @@ existing multi-key failover runs. Default off → zero behavior change for exist dropped, never a config-rejecting error; the outer provider schema stays passthrough). - `src/providers/key-failover.ts`: `rateLimitRetryPolicyFor` (normalize/default) and `rateLimitRetryDelayMs` (Retry-After seconds/HTTP-date → capped, else `intervalMs`), - reusing the existing `parseRetryAfterMs` cooldown parser. Key-auth providers only: OAuth and - forward credentials are never replayed on the same token. + reusing the existing `parseRetryAfterMs` cooldown parser; the fixed fallback is capped at + `maxIntervalMs` too, so a single wait never exceeds the cap. API-key providers only + (`authMode: "key"`): OAuth/forward credentials are never replayed on the same token, and + local runtimes have no remote key to preserve. - `src/usage/log.ts`: new `AttemptRecoveryKind` member `"rate-limit-429"`. - `src/server/responses/core.ts`: in the pre-stream recovery loop, BEFORE the multi-key failover `while`, wait then `rebuildAndRefetch("rate-limit-429")`. Abort during the wait cancels the client request (the unread 429 body is released first). The retry budget lives OUTSIDE the recovery loop, so a 413/401 replay that comes back 429 cannot re-arm a fresh budget — bounded to `attempts` per request. After attempts are exhausted the existing - failover and error mapping run unchanged. Covers Responses, chat completions, and routed - Claude messages (they all enter `handleResponses`). + failover and error mapping run unchanged. The same wait-and-replay applies to the other + key-auth surfaces that bypass that loop: + - Responses passthrough wire (`openai-responses` key-auth gateways, e.g. the built-in + DeepSeek preset) — pre-relay, before the forward-pool logic; + - image/video bridge and web-search sidecar loops (`src/images/loop.ts`, + `src/web-search/loop.ts`) — before their `on429` key rotation; + - Anthropic terminal-guard continuations — before key/account failover. + Covers Responses, chat completions, and routed Claude messages (they all enter + `handleResponses`). ## Safety @@ -68,4 +77,9 @@ existing multi-key failover runs. Default off → zero behavior change for exist - `tests/usage-log.test.ts` — the `rate-limit-429` recovery kind survives persisted usage logs. - `tests/server-rate-limit-retry-e2e.test.ts` — single-key replay to success, immediate passthrough without the knob, exhausted attempts surface 429, and retry-before-failover - ordering with a 2-key pool. + ordering with a 2-key pool, plus key-auth `openai-responses` passthrough replaying 429 on + the same key. +- `tests/terminal-guard-server.test.ts` — an Anthropic terminal-guard continuation that 429s + is replayed on the same key before the error surfaces (3 upstream sends). +- `tests/images/loop.test.ts` + `tests/web-search.test.ts` — the bridge loops replay 429 on + the same key before `on429` rotation runs (same-key sends counted, rotations zero). diff --git a/docs-site/src/content/docs/ja/reference/configuration.md b/docs-site/src/content/docs/ja/reference/configuration.md index bc975569a..e110a697f 100644 --- a/docs-site/src/content/docs/ja/reference/configuration.md +++ b/docs-site/src/content/docs/ja/reference/configuration.md @@ -194,7 +194,7 @@ timing side channel を防ぐため定数時間(`timingSafeEqual`)で比較 | `parallelToolCalls?` | `boolean` | 並列ツール呼び出しをオン/オフします。OpenAI Chat はデフォルト on で、chat 以外のアダプターは明示的な `true` でのみサポートを公表します。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` で `auto` または `none` だけを受け付けるモデル。強制/指定選択は downgrade します。 | | `preserveReasoningContentModels?` | `string[]` | 前の assistant `reasoning_content` を chat history に維持すべきモデル。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | key-auth プロバイダーのみ。オプトインの同一ターゲット 429 リトライ: 待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は 600000 で上限)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `thinkingToggleModels?` | `string[]` | effort 段階の代わりに vendor `thinking.enabled` toggle を使う chat モデル。 | | `thinkingBudgetModels?` | `string[]` | 整数 `thinking_budget` を使う chat モデル。effort を budget 比率にマッピングします。 | | `noVisionModels?` | `string[]` | テキスト専用モデル。[ビジョンサイドカー](/ja/guides/sidecars/) が画像を説明します。Ollama の `:size` タグも一致させます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration.md b/docs-site/src/content/docs/ko/reference/configuration.md index 42529163c..90d925007 100644 --- a/docs-site/src/content/docs/ko/reference/configuration.md +++ b/docs-site/src/content/docs/ko/reference/configuration.md @@ -204,7 +204,7 @@ timing side channel을 막기 위해 상수 시간(`timingSafeEqual`)으로 비 | `parallelToolCalls?` | `boolean` | 병렬 툴 호출을 켜거나 끕니다. OpenAI Chat은 기본 on이며, chat 외 어댑터는 명시적인 `true`에서만 지원을 알립니다. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`에서 `auto` 또는 `none`만 받는 모델. 강제/지정 선택은 downgrade합니다. | | `preserveReasoningContentModels?` | `string[]` | 이전 assistant `reasoning_content`를 chat history에 유지해야 하는 모델. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | key-auth 프로바이더 전용. 동일 대상 429 재시도: 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 600000으로 상한), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | effort 단계 대신 vendor `thinking.enabled` toggle을 쓰는 chat 모델. | | `thinkingBudgetModels?` | `string[]` | 정수 `thinking_budget`을 쓰는 chat 모델. effort를 budget 비율로 매핑합니다. | | `noVisionModels?` | `string[]` | 텍스트 전용 모델. [비전 사이드카](/ko/guides/sidecars/)가 이미지를 설명합니다. Ollama의 `:size` 태그도 일치시킵니다. | diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.md index ac48b610f..f331a62b8 100644 --- a/docs-site/src/content/docs/reference/configuration.md +++ b/docs-site/src/content/docs/reference/configuration.md @@ -350,7 +350,7 @@ or bind the forward explicitly to loopback (`ssh -L 127.0.0.1:20100:localhost:10 | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Provider-local, disabled-by-default passthrough SSE repair for exact `message` / `reasoning` placeholder ids and missing terminal ids. Downstream only; function-call ids and `call_id` are never rewritten. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced/named choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models that require prior assistant `reasoning_content` to remain in chat history. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Key-auth providers only. Opt-in same-target 429 retry: wait (upstream `Retry-After` or the fixed interval) and replay the identical request on the same key before any key failover. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (a single wait is capped at 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | Chat models using a vendor `thinking.enabled` toggle instead of an effort ladder. | | `thinkingBudgetModels?` | `string[]` | Chat models using an integer `thinking_budget`; effort is mapped to a budget fraction. | | `noVisionModels?` | `string[]` | Text-only models — the [vision sidecar](/guides/sidecars/) describes images for them. Matching tolerates an Ollama `:size` tag. | diff --git a/docs-site/src/content/docs/ru/reference/configuration.md b/docs-site/src/content/docs/ru/reference/configuration.md index 80a3c01b5..d7c9762be 100644 --- a/docs-site/src/content/docs/ru/reference/configuration.md +++ b/docs-site/src/content/docs/ru/reference/configuration.md @@ -228,7 +228,7 @@ Responses и Chat Completions принимают только выделенны | `parallelToolCalls?` | `boolean` | Включает/отключает параллельные вызовы инструментов. В OpenAI Chat включено по умолчанию; не-chat-адаптеры объявляют поддержку только при явном `true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; принудительные/именованные выборы понижаются. | | `preserveReasoningContentModels?` | `string[]` | Модели, требующие сохранять предыдущий assistant `reasoning_content` в истории чата. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с key-auth. Опциональный повтор при 429 на том же таргете: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (одно ожидание ограничено 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие вендорский переключатель `thinking.enabled` вместо лестницы уровней рассуждений. | | `thinkingBudgetModels?` | `string[]` | Chat-модели, использующие целочисленный `thinking_budget`; уровень отображается в долю бюджета. | | `noVisionModels?` | `string[]` | Модели только для текста — [vision-сайдкар](/ru/guides/sidecars/) описывает для них изображения. Сопоставление допускает тег Ollama `:size`. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration.md b/docs-site/src/content/docs/zh-cn/reference/configuration.md index 5ec692b55..39055a09c 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration.md @@ -196,7 +196,7 @@ Codex Direct 透传,两个 bearer 域不能混淆。仪表盘的 API 标签页 | `parallelToolCalls?` | `boolean` | 启用或禁用并行工具调用。OpenAI Chat 默认开启;非 chat adapter 只有显式为 `true` 时才公布支持。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型;forced/named 选择会降级。 | | `preserveReasoningContentModels?` | `string[]` | 要求在 chat history 中保留先前 assistant `reasoning_content` 的模型。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 key-auth 提供商。可选的同目标 429 重试:等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待上限 600000)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | | `thinkingToggleModels?` | `string[]` | 使用 vendor `thinking.enabled` toggle,而不是 effort ladder 的 chat 模型。 | | `thinkingBudgetModels?` | `string[]` | 使用整数 `thinking_budget` 的 chat 模型;effort 会映射成 budget 比例。 | | `noVisionModels?` | `string[]` | 纯文本模型;[视觉 sidecar](/zh-cn/guides/sidecars/) 会为它们描述图像。匹配时容忍 Ollama `:size` 标签。 | diff --git a/src/images/loop.ts b/src/images/loop.ts index 535f295df..0c020c6ca 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -14,12 +14,13 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../adapters/ import { existsSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { createAdapterEventQueue } from "../adapters/run-turn-queue"; -import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderContinuationState, OcxRequestOptions, OcxThinkingContent, OcxUsage } from "../types"; +import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderContinuationState, OcxRequestOptions, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types"; import { namespacedToolName } from "../types"; import { bridgeToResponsesSSE } from "../bridge"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry } from "../lib/upstream-retry"; +import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry"; +import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, TRANSLATOR_MAX_TURN_BYTES, @@ -233,6 +234,8 @@ export interface ImageBridgeDeps { * rotated key, or null when the pool is exhausted. */ on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; + /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ + retryOn429Policy?: Required | null; /** Called when the bridged Responses stream completes (parity with runTurn / routed paths). */ onCompletedResponse?: (response: Record, providerState?: OcxProviderContinuationState) => void; /** WebSocket Responses path only — leave response id empty for protocol compatibility. */ @@ -461,6 +464,30 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } + throw new LoopError(499, "client closed request during image-bridge"); + } + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // Stall-watchdog seam between bounded retry fetches. + yield { type: "heartbeat" }; + prepared = await fetchOnce(adapter); + } // 429 key-failover parity with web-search / normal routed path. while (prepared.response.status === 429 && deps.on429) { const rotated = deps.on429(prepared.response.headers.get("retry-after")); diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 139807f9b..e6ca014ef 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -76,15 +76,16 @@ export function hasKeyPoolFailover(provider: OcxProviderConfig): boolean { /** * Normalize a provider's `retryOn429` policy, or return null when the knob is absent, * explicitly disabled, or the provider is not key-auth (OAuth/forward credentials must not be - * replayed on the same token, and forward passthrough never reaches the recovery loop anyway). - * The returned policy is fully defaulted so callers never re-check fields. + * replayed on the same token, forward passthrough never reaches the recovery loop anyway, and + * local runtimes have no remote key to preserve). The returned policy is fully defaulted so + * callers never re-check fields. */ export function rateLimitRetryPolicyFor( provider: Pick, ): Required | null { const policy = provider.retryOn429; if (!policy || policy.enabled === false) return null; - if (provider.authMode === "oauth" || provider.authMode === "forward") return null; + if (provider.authMode === "oauth" || provider.authMode === "forward" || provider.authMode === "local") return null; return { enabled: policy.enabled ?? DEFAULT_RATE_LIMIT_RETRY.enabled, attempts: policy.attempts ?? DEFAULT_RATE_LIMIT_RETRY.attempts, @@ -97,7 +98,8 @@ export function rateLimitRetryPolicyFor( /** * Wait before the next same-target replay: upstream Retry-After (seconds or HTTP-date) when * `respectRetryAfter` is on and the header parses, capped at `maxIntervalMs`; otherwise the - * fixed `intervalMs`. Malformed headers fall back to the fixed interval. + * fixed `intervalMs`, also capped at `maxIntervalMs` (a single wait never exceeds the cap). + * Malformed headers fall back to the fixed interval. */ export function rateLimitRetryDelayMs( policy: Required, @@ -109,7 +111,7 @@ export function rateLimitRetryDelayMs( const parsed = parseRetryAfterMs(raw, now); if (parsed !== undefined) return Math.min(parsed, policy.maxIntervalMs); } - return policy.intervalMs; + return Math.min(policy.intervalMs, policy.maxIntervalMs); } /** diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 7bbeb622c..b06a00694 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1628,6 +1628,48 @@ async function handleResponsesInner( request.releaseBodyObservation?.(); } + // Same-target 429 wait-and-retry (opt-in `retryOn429`) for key-auth providers on the + // passthrough wire. This branch returns before the recovery loop below, so Responses-shaped + // key-auth gateways (e.g. the built-in DeepSeek preset) would otherwise surface 429 + // immediately with no same-key replay. Pre-stream only — nothing has been relayed yet, so + // the replay is lossless (same invariant as the recovery loop). Forward/OAuth providers + // keep their pool logic below (rateLimitRetryPolicyFor returns null for them). + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; + while ( + upstreamResponse.status === 429 + && rateLimitPolicy !== null + && rateLimitRetries < rateLimitPolicy.attempts + ) { + rateLimitRetries += 1; + try { + await sleepWithAbort( + rateLimitRetryDelayMs(rateLimitPolicy, upstreamResponse.headers.get("retry-after"), Date.now()), + options.abortSignal, + ); + } catch { + cancelResponseBodyBestEffort(upstreamResponse); + upstream.abort(); + return clientCancelledResponse(); + } + cancelResponseBodyBestEffort(upstreamResponse); + try { + upstreamResponse = await fetchWithTransientRetry( + recovery => { + noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery); + return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({ + method: request.method, + headers: request.headers, + body: request.body, + }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)); + }, + { abortSignal: upstream.signal, label: safeHostLabel(request.url) }, + ); + } catch (err) { + return transportFailureResponse(err); + } + } + if (usesCodexForwardPoolAuth(authCtx, route.provider)) { let poolRetryOutcome: number | undefined; if (await shouldRetryCodexPoolAccountModel400( @@ -2048,6 +2090,7 @@ async function handleResponsesInner( config.cacheRetention, ); }, + retryOn429Policy: rateLimitRetryPolicyFor(route.provider), ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), ...(options.forceEmptyResponseId ? { forceEmptyResponseId: true } : {}), onCompletedResponse: (response, providerState) => @@ -2116,6 +2159,7 @@ async function handleResponsesInner( config.cacheRetention, ); }, + retryOn429Policy: rateLimitRetryPolicyFor(route.provider), }); // Register the sidecar stream as an active turn so drainAndShutdown waits for (or aborts) // in-flight web-search turns instead of skipping them during graceful shutdown. @@ -2528,49 +2572,51 @@ async function handleResponsesInner( const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let imageTierBias = 0; let response: Response | undefined; - while (true) { + const fetchContinuation = async (): Promise => { + const continuationRequest = await activeAdapter.buildRequest(nextParsed, { + headers: selectedForwardHeaders, + translatorBudget, + ...(imageTierBias > 0 ? { imageTierBias } : {}), + }); + recordAdapterReasoning(logCtx, continuationRequest); + const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number" + ? continuationRequest.usageLog.inputTokens + : undefined; + if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; try { - const continuationRequest = await activeAdapter.buildRequest(nextParsed, { - headers: selectedForwardHeaders, - translatorBudget, - ...(imageTierBias > 0 ? { imageTierBias } : {}), - }); - recordAdapterReasoning(logCtx, continuationRequest); - const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number" - ? continuationRequest.usageLog.inputTokens - : undefined; - if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; - try { - if (activeAdapter.fetchResponse) { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate); - response = await activeAdapter.fetchResponse(continuationRequest, { - abortSignal: upstream.signal, - timeoutMs: connectMs, - stream: nextParsed.stream, - }); - } else { - response = await fetchWithResetRetry( - recovery => { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery); - return fetchWithHeaderTimeout( - continuationRequest.url, - applyUpstreamRecoveryInit({ - method: continuationRequest.method, - headers: continuationRequest.headers, - body: continuationRequest.body, - }, recovery), - upstream.signal, - connectMs, - nextParsed.stream, - providerFetch(route.provider), - ); - }, - { abortSignal: upstream.signal, label: safeHostLabel(continuationRequest.url) }, - ); - } - } finally { - continuationRequest.releaseBodyObservation?.(); + if (activeAdapter.fetchResponse) { + noteAttemptSend(logCtx.activeAttempt, continuationEstimate); + return await activeAdapter.fetchResponse(continuationRequest, { + abortSignal: upstream.signal, + timeoutMs: connectMs, + stream: nextParsed.stream, + }); } + return await fetchWithResetRetry( + recovery => { + noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery); + return fetchWithHeaderTimeout( + continuationRequest.url, + applyUpstreamRecoveryInit({ + method: continuationRequest.method, + headers: continuationRequest.headers, + body: continuationRequest.body, + }, recovery), + upstream.signal, + connectMs, + nextParsed.stream, + providerFetch(route.provider), + ); + }, + { abortSignal: upstream.signal, label: safeHostLabel(continuationRequest.url) }, + ); + } finally { + continuationRequest.releaseBodyObservation?.(); + } + }; + while (true) { + try { + response = await fetchContinuation(); } catch (error) { if (options.abortSignal?.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; @@ -2580,6 +2626,44 @@ async function handleResponsesInner( return; } + // Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover: + // a primary-key rate-limit blip replays on the SAME key, matching the main recovery + // loop; only after the attempts are exhausted does the continuation fail over. + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; + while ( + response.status === 429 + && rateLimitPolicy !== null + && rateLimitRetries < rateLimitPolicy.attempts + ) { + rateLimitRetries += 1; + try { + await sleepWithAbort( + rateLimitRetryDelayMs(rateLimitPolicy, response.headers.get("retry-after"), Date.now()), + options.abortSignal, + ); + } catch { + try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + if (options.abortSignal?.aborted) { + yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; + } else { + yield { type: "error", message: "Provider continuation failed: retry wait interrupted" }; + } + return; + } + try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + try { + response = await fetchContinuation(); + } catch (error) { + if (options.abortSignal?.aborted) { + yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; + } else { + yield { type: "error", message: `Provider continuation failed: ${error instanceof Error ? error.message : String(error)}` }; + } + return; + } + } + if (response.status === 429 && hasKeyPoolFailover(route.provider)) { const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter: response.headers.get("retry-after"), diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index b4a5e2797..b220fbf0e 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -1,5 +1,5 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../adapters/base"; -import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxThinkingContent, OcxUsage } from "../types"; +import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types"; import { namespacedToolName } from "../types"; import { bridgeToResponsesSSE } from "../bridge"; import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor"; @@ -7,7 +7,8 @@ import { runAnthropicWebSearch } from "./anthropic-executor"; import { clearableDeadline } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry } from "../lib/upstream-retry"; +import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry"; +import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, TRANSLATOR_MAX_TURN_BYTES, @@ -244,6 +245,8 @@ export interface WebSearchLoopDeps { * or null when the pool is exhausted (same semantics as the normal routed path). */ on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; + /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ + retryOn429Policy?: Required | null; } /** @@ -355,6 +358,30 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } + throw new LoopError(499, "client closed request during web-search"); + } + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // Stall-watchdog seam between bounded retry fetches. + yield { type: "heartbeat" }; + prepared = await fetchOnce(adapter); + } // 429 key-failover parity with the normal routed path: rotate pool keys until one responds // or the pool is exhausted (deps.on429 returns null — cooldown map guarantees termination). while (prepared.response.status === 429 && deps.on429) { diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index 359de4615..d0142ab84 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -220,14 +220,18 @@ the original user/tool-result turn for reasoning-only attempts, supplies neutral for empty tool output, and validates role alternation plus tool-use/result pairing before transport. Provider-level `retryOn429` (devlog 260802_429_same_target_retry) is the generic, opt-in -same-target 429 retry for key-auth providers, primarily single-key pools that cannot use -multi-key failover. In the pre-stream recovery loop, a 429 waits (`Retry-After` or the fixed -interval, capped) and replays the identical request on the same key before any failover, up to -`attempts` extra times per request (the budget lives outside the recovery loop, so a 413/401 -replay cannot re-arm it). Codex never retries 429 client-side (openai/codex#30471), so this is -the only defense for those providers; the final 429 still carries `Retry-After` for clients that -honor it. Concurrent requests each honor their own policy — there is no process-wide shared -cooldown (unlike the Kiro pattern), so a rate-limit storm multiplies upstream volume by at most +same-target 429 retry for API-key providers (`authMode: "key"`), primarily single-key pools +that cannot use multi-key failover. In the pre-stream recovery loop, a 429 waits (`Retry-After` +or the fixed interval, capped at `maxIntervalMs`) and replays the identical request on the same +key before any failover, up to `attempts` extra times per request (the budget lives outside the +recovery loop, so a 413/401 replay cannot re-arm it). The same wait-and-replay applies to every +other key-auth surface that bypasses that loop: the Responses passthrough wire (e.g. the +built-in DeepSeek preset), the image/video bridge and web-search sidecar loops (before their +`on429` key rotation), and Anthropic terminal-guard continuations (before key/account +failover). Codex never retries 429 client-side (openai/codex#30471), so this is the only +defense for those providers; the final 429 still carries `Retry-After` for clients that honor +it. Concurrent requests each honor their own policy — there is no process-wide shared cooldown +(unlike the Kiro pattern), so a rate-limit storm multiplies upstream volume by at most `attempts + 1` per request. The wait is abort-aware: once the server observes the client disconnect (Bun propagates it asynchronously, observed 1–10 s), the sleep is interrupted, the unread 429 body is released, and the request is cancelled with 499 before any replay; because diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index 6795e8873..132ef0d4f 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -177,6 +177,39 @@ describe("runWithImageBridge", () => { expect(buildRequestCalls).toBe(1); }); + test("retryOn429 replays on the same key before on429 rotation", async () => { + let sends = 0; + let rotations = 0; + const retryingAdapter: ProviderAdapter = { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + }, + }; + streamQueue = [[{ type: "text_delta" as const, text: "recovered" }, { type: "done" as const }]]; + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: retryingAdapter, + plan, + retryOn429Policy: { enabled: true, attempts: 2, intervalMs: 120, maxIntervalMs: 60_000, respectRetryAfter: false }, + on429: () => { + rotations += 1; + return null; + }, + }); + const sse = await response.text(); + expect(sse).toContain("recovered"); + expect(sends).toBe(2); + expect(rotations).toBe(0); + }); + test("forced-final clears named image tool_choice", async () => { streamQueue = [ [{ type: "text_delta" as const, text: "done" }, { type: "done" as const }], diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts index f86f8e4e8..75e77dba3 100644 --- a/tests/rate-limit-retry.test.ts +++ b/tests/rate-limit-retry.test.ts @@ -12,7 +12,7 @@ describe("rateLimitRetryPolicyFor", () => { expect(rateLimitRetryPolicyFor({ retryOn429: { enabled: false } } as OcxProviderConfig)).toBeNull(); }); - test("null for OAuth and forward providers (same-token replays are never attempted)", () => { + test("null for OAuth, forward, and local providers (no remote key to preserve)", () => { expect(rateLimitRetryPolicyFor({ authMode: "oauth", retryOn429: {}, @@ -21,6 +21,10 @@ describe("rateLimitRetryPolicyFor", () => { authMode: "forward", retryOn429: {}, } as OcxProviderConfig)).toBeNull(); + expect(rateLimitRetryPolicyFor({ + authMode: "local", + retryOn429: {}, + } as OcxProviderConfig)).toBeNull(); expect(rateLimitRetryPolicyFor({ authMode: "key", retryOn429: {}, @@ -72,6 +76,14 @@ describe("rateLimitRetryDelayMs", () => { expect(rateLimitRetryDelayMs(policy, "0", 1_000_000)).toBe(1); }); + test("fixed fallback is capped at maxIntervalMs (a single wait never exceeds the cap)", () => { + const p = rateLimitRetryPolicyFor({ + retryOn429: { intervalMs: 600_000, maxIntervalMs: 100 }, + } as OcxProviderConfig)!; + expect(rateLimitRetryDelayMs(p, null, 1_000_000)).toBe(100); + expect(rateLimitRetryDelayMs(p, "3600", 1_000_000)).toBe(100); + }); + test("malformed Retry-After falls back to the fixed interval", () => { expect(rateLimitRetryDelayMs(policy, "soon", 1_000_000)).toBe(5_000); }); diff --git a/tests/server-rate-limit-retry-e2e.test.ts b/tests/server-rate-limit-retry-e2e.test.ts index 414a7ecca..1811e6148 100644 --- a/tests/server-rate-limit-retry-e2e.test.ts +++ b/tests/server-rate-limit-retry-e2e.test.ts @@ -233,4 +233,63 @@ describe("server same-target 429 retry (end-to-end)", () => { globalThis.fetch = originalFetch; } }); + + test("key-auth openai-responses passthrough replays 429 on the same key", async () => { + const originalFetch = globalThis.fetch; + let sends = 0; + const seenAuth: string[] = []; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://passthrough.test/v1/responses") { + sends += 1; + const auth = new Headers(init?.headers).get("authorization") ?? ""; + seenAuth.push(auth); + if (sends === 1) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return new Response(JSON.stringify({ + id: "resp-ok", + object: "response", + status: "completed", + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "ok after retry" }] }], + }), { status: 200, headers: { "content-type": "application/json" } }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "passthrough", + providers: { + passthrough: { + adapter: "openai-responses", + baseUrl: "https://passthrough.test/v1", + authMode: "key", + apiKey: "key-alpha-000111222333", + retryOn429: { attempts: 2, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "passthrough/model"); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).toContain("ok after retry"); + expect(sends).toBe(2); + expect(seenAuth).toEqual([ + "Bearer key-alpha-000111222333", + "Bearer key-alpha-000111222333", + ]); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); }); diff --git a/tests/terminal-guard-server.test.ts b/tests/terminal-guard-server.test.ts index f85a3cddf..fe246fd0a 100644 --- a/tests/terminal-guard-server.test.ts +++ b/tests/terminal-guard-server.test.ts @@ -78,4 +78,47 @@ describe("server terminal guard integration", () => { expect(messages.at(-1)?.content?.[0]?.text).toContain("你刚才只描述了计划"); }); + test("terminal-guard continuation 429 replays on the same key before surfacing", async () => { + const retryConfig = { + ...config, + providers: { + "claude-se": { + adapter: "anthropic", + baseUrl: "https://example.test", + apiKey: "sk-test", + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as unknown as OcxConfig; + let sends = 0; + globalThis.fetch = (async (_input, init) => { + sends += 1; + if (sends === 2) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return anthropicSse(sends === 1 ? firstTurn : continuationTurn); + }) as typeof fetch; + + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "se-claude-opus-4.8", + input: "请检查这个问题并修复代码", + stream: true, + tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }], + }), + }), retryConfig, { model: "", provider: "" }); + + const text = await response.text(); + expect(response.status).toBe(200); + // initial turn + 429 continuation + replayed continuation + expect(sends).toBe(3); + expect(text).toContain("response.completed"); + expect(text).toContain("exec_command"); + }); + }); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index 6b6ebe8f0..bbed3778f 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -623,6 +623,59 @@ describe("web-search sidecar native web_search_call emission", () => { ]); }); + test("retryOn429 replays on the same key before on429 rotation", async () => { + globalThis.fetch = (() => Promise.resolve(new Response( + 'event: response.completed\ndata: {"type":"response.completed"}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ))) as typeof fetch; + + let sends = 0; + let rotations = 0; + const retryingAdapter: ProviderAdapter = { + name: "mock-retry429", + buildRequest: () => ({ + url: "https://routed.test/v1", + method: "POST", + headers: {}, + body: "{}", + }), + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response("rate limited", { status: 429, headers: { "retry-after": "30" } }); + } + return new Response("{}", { status: 200 }); + }, + async *parseStream() { + yield { type: "text_delta", text: "answer after same-key retry" }; + yield { type: "done" }; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter: retryingAdapter, + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.4-mini", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + retryOn429Policy: { enabled: true, attempts: 2, intervalMs: 120, maxIntervalMs: 60_000, respectRetryAfter: false }, + on429: () => { + rotations += 1; + return null; + }, + }); + expect(response.status).toBe(200); + const frames = await collectSse(response.body!); + const completed = frames.find(f => f.event === "response.completed")?.data.response as Record; + const output = completed.output as { type: string; content?: { text?: string }[] }[]; + expect(output.find(o => o.type === "message")?.content?.[0]?.text).toBe("answer after same-key retry"); + expect(sends).toBe(2); + expect(rotations).toBe(0); + }); + test("loop 429 with exhausted pool (on429 null) surfaces the provider error", async () => { const firstAdapter: ProviderAdapter = { name: "mock-429", From 58c36c9e9fcc7367e620ba75a8ae0a7bad42447a Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 03:19:05 +0800 Subject: [PATCH 04/29] fix(proxy): resolve review-bot round on retryOn429 (fail-closed auth, deadlines, recovery labels) - key-failover: fail closed - only authMode key (or the documented omitted default) may replay; unknown modes rejected; an already-expired HTTP-date Retry-After retries immediately (like Retry-After: 0) - derive: providerConfigSeed preserves the registry auth kind (incl. local) so the gate survives the seed round-trip and routing - core: release the unread 429 body BEFORE every backoff (main loop, passthrough, continuations); passthrough and continuation replay sends record the rate-limit-429 recovery kind; continuation budget hoisted outside the failover loop so rotation/413 cannot re-arm it; continuation waits sleep on the upstream signal so an SSE body-cancel aborts them too - images/web-search loops: release the 429 body before the wait, restart the response-header deadline after each deliberate wait (backoffs never consume the connect budget or surface as 504), and record rate-limit-429 on replay sends - config: load-time degradation for invalid optional retryOn429 fields (warn + drop the field) instead of tripping the whole schema and hiding all providers behind a default config; the management write boundary still rejects - tests: fail-closed auth modes (incl. unknown), expired HTTP-date, per-request budget across 2-key pools (e2e + terminal continuation), byte-identical replay bodies/auth (passthrough + continuation), recovery telemetry in both loops, config load degradation, registry auth-kind preservation, typed usage-log entry - docs: retry-wait vs total-latency and attempts+poolKeys volume bounds, identical-replay equivalence, deadline/backoff behavior in devlog 010 and structure/04; retryOn429 boundary notes in the provider guide and adapter reference (English + ja/ko/ru/zh-cn) --- .../010_design.md | 43 +++++++++++---- .../src/content/docs/guides/providers.md | 5 ++ .../src/content/docs/ja/guides/providers.md | 4 ++ .../src/content/docs/ja/reference/adapters.md | 4 ++ .../src/content/docs/ko/guides/providers.md | 4 ++ .../src/content/docs/ko/reference/adapters.md | 4 ++ .../src/content/docs/reference/adapters.md | 5 ++ .../src/content/docs/ru/guides/providers.md | 5 ++ .../src/content/docs/ru/reference/adapters.md | 4 ++ .../content/docs/zh-cn/guides/providers.md | 4 ++ .../content/docs/zh-cn/reference/adapters.md | 4 ++ src/config.ts | 40 ++++++++++++++ src/images/loop.ts | 17 ++++-- src/providers/derive.ts | 4 +- src/providers/key-failover.ts | 10 +++- src/server/responses/core.ts | 51 +++++++++++------- src/web-search/loop.ts | 17 ++++-- structure/04_transports-and-sidecars.md | 13 +++-- tests/config-user-edits.test.ts | 35 +++++++++++++ tests/images/loop.test.ts | 5 ++ tests/provider-registry-parity.test.ts | 9 ++++ tests/rate-limit-retry.test.ts | 12 ++++- tests/server-rate-limit-retry-e2e.test.ts | 52 +++++++++++++++++++ tests/terminal-guard-server.test.ts | 50 ++++++++++++++++++ tests/usage-log.test.ts | 6 ++- tests/web-search.test.ts | 5 ++ 26 files changed, 366 insertions(+), 46 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index 7a0c0542f..3a13fe481 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -26,12 +26,17 @@ existing multi-key failover runs. Default off → zero behavior change for exist - `src/types.ts`: `RateLimitRetryPolicy` interface + `OcxProviderConfig.retryOn429`. - `src/config.ts`: zod schema entry (zod's default strip inside the object — an unknown key is dropped, never a config-rejecting error; the outer provider schema stays passthrough). + Load-time degradation: one hand-edited invalid optional field (e.g. `attempts: 0`) is dropped + with a warning instead of tripping the whole schema and hiding all providers behind a default + config; the management write boundary still rejects invalid policies. - `src/providers/key-failover.ts`: `rateLimitRetryPolicyFor` (normalize/default) and `rateLimitRetryDelayMs` (Retry-After seconds/HTTP-date → capped, else `intervalMs`), reusing the existing `parseRetryAfterMs` cooldown parser; the fixed fallback is capped at - `maxIntervalMs` too, so a single wait never exceeds the cap. API-key providers only - (`authMode: "key"`): OAuth/forward credentials are never replayed on the same token, and - local runtimes have no remote key to preserve. + `maxIntervalMs` too, so a single wait never exceeds the cap. Fail closed: only `authMode: + "key"` (or the documented omitted default for custom API-key providers) may use replays — + OAuth/forward are never replayed on the same token, local runtimes have no remote key to + preserve, and unknown values are rejected. `providerConfigSeed` preserves the registry auth + kind (including `"local"`) so the gate survives the seed round-trip. - `src/usage/log.ts`: new `AttemptRecoveryKind` member `"rate-limit-429"`. - `src/server/responses/core.ts`: in the pre-stream recovery loop, BEFORE the multi-key failover `while`, wait then `rebuildAndRefetch("rate-limit-429")`. Abort during the wait @@ -45,6 +50,9 @@ existing multi-key failover runs. Default off → zero behavior change for exist - image/video bridge and web-search sidecar loops (`src/images/loop.ts`, `src/web-search/loop.ts`) — before their `on429` key rotation; - Anthropic terminal-guard continuations — before key/account failover. + Every surface releases the unread 429 body BEFORE the backoff, records the + `rate-limit-429` recovery kind on replay sends, and (bridges) restarts the response-header + deadline after each deliberate wait. Covers Responses, chat completions, and routed Claude messages (they all enter `handleResponses`). @@ -54,17 +62,34 @@ existing multi-key failover runs. Default off → zero behavior change for exist request is lossless (same invariant as the transient-5xx layer in `lib/upstream-retry.ts`). - Ordering: same-key retries run before failover, so "primary-first" users keep their key on rate-limit blips; failover still works after retries exhaust. -- Latency bound: worst case `attempts × maxIntervalMs` (default 3 × 60 s = 180 s) when - honoring upstream `Retry-After`; `attempts × intervalMs` (default 15 s) when - `respectRetryAfter=false` or no header is present. +- Retry-wait bound: the SLEEP component is at most `attempts × maxIntervalMs` (default + 3 × 60 s = 180 s) when honoring upstream `Retry-After`; `attempts × intervalMs` + (default 15 s) when `respectRetryAfter=false` or no header is present. Total request + latency is higher: every attempt also consumes its own connect/response time (bounded by + `connectTimeoutMs`), so the documented bound covers deliberate waits only. +- Identical replay: rebuilds are deterministic for the same parsed request (same serialized + body and auth headers); the passthrough/continuation/e2e tests assert byte-identical bodies + and identical auth headers across replays, not just send counts. - Abort during the wait: the sleep is abort-aware — when the server observes the client disconnect (Bun propagates this asynchronously, observed 1–10 s), the wait is interrupted, the unread 429 body is released, and the request is cancelled with 499 before any replay. Because the propagation is async, a replay can still precede the cancel if the interval - elapses first; that is bounded by the same `attempts` budget. + elapses first; that is bounded by the same `attempts` budget. Terminal continuations sleep + on the upstream signal, so a body-cancel (SSE already streaming) aborts the wait too. - Concurrency: each request honors its own policy independently — no process-wide cooldown is - shared between concurrent requests (unlike the Kiro 429 pattern). Opt-in and bounded, so a - storm multiplies upstream volume by at most `attempts + 1` per request. + shared between concurrent requests (unlike the Kiro 429 pattern). Upstream volume per + request: same-key replays add at most `attempts` sends, then multi-key failover adds up to + `poolKeys − 1` more (or Anthropic account rotations), so the combined bound is + `attempts + poolKeys` sends — a storm multiplies by that factor per request, not by + `attempts + 1`. +- Header deadlines: the image/video and web-search bridge loops restart their response-header + deadline after each deliberate wait, so backoffs never consume the connect budget and a + rate-limit wait is never misattributed as a 504 header timeout. +- Expired `Retry-After`: a valid HTTP-date already in the past retries immediately (same as + numeric `Retry-After: 0`) instead of falling back to the fixed interval. +- Recovery observability: every retry surface records the `rate-limit-429` recovery kind + (normal loop, passthrough wire, image/video bridge, web-search sidecar, terminal + continuations), so usage logs explain the extra sends. - Final 429 still carries `Retry-After` for clients that honor it (Claude Code). ## Tests diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index 2c3da6070..cc2b094bc 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -39,6 +39,11 @@ labels local presets separately; those normally omit both `authMode` and `apiKey | `forward` | Relays **your incoming Codex auth headers** verbatim to the provider — no key stored. This is the ChatGPT-login passthrough. | OpenAI (`openai-responses` adapter). | | `oauth` | Resolves a stored OAuth access token (auto-refreshed before expiry) and uses it as the bearer key. | xAI, Anthropic, Kimi, Kiro, Google Antigravity, Cursor, GitHub Copilot. | +The [`retryOn429`](/reference/configuration/) same-key 429 replay applies only to API-key +providers (`authMode: "key"`). OAuth, forward, and local presets are excluded — their +credentials must never be replayed on the same token, and local runtimes have no remote key to +preserve. + ## 1. ChatGPT login (forward / passthrough) The `openai` provider needs **no API key**. Direct forwards credentials from your existing diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index b47146a0c..bbc9dba31 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -35,6 +35,10 @@ max input 922,000 で `*-pro` virtual ID は公開状態を維持し、wire で | `forward` | **受け取った Codex 認証ヘッダーを**プロバイダーにそのまま中継します — キーを保存しません。ChatGPT ログインのパススルーです。 | OpenAI(`openai-responses` アダプター)。 | | `oauth` | 保存された OAuth アクセストークンを読み込み bearer キーとして使い、期限切れ前に自動更新します。 | xAI、Anthropic、Kimi、Kiro、Google Antigravity、Cursor。 | +[`retryOn429`](/ja/reference/configuration/)(同一キーでの 429 リトライ)は API キー プロバイダー +(`authMode: "key"`)のみに適用されます。OAuth・forward・ローカル プリセットは除外されます — +同じトークンを再送すべきではなく、ローカルランタイムには保存すべきリモートキーがありません。 + ## 1. ChatGPT ログイン(forward / パススルー) デフォルトプロバイダーは**API キー不要**です。既存の `codex login` の認証情報を OpenAI Responses バックエンドに diff --git a/docs-site/src/content/docs/ja/reference/adapters.md b/docs-site/src/content/docs/ja/reference/adapters.md index ea97ddbcf..4276d9b62 100644 --- a/docs-site/src/content/docs/ja/reference/adapters.md +++ b/docs-site/src/content/docs/ja/reference/adapters.md @@ -38,6 +38,10 @@ interface ProviderAdapter { **対象:** OpenAI **Responses API**。**`passthrough: true`** — 元のリクエスト本文をそのまま渡し、レスポンスを **変換せずに** ストリーミングします。 **認証:** `forward`(呼び出し元ヘッダー中継)または `key`。 +`key` 認証では、[`retryOn429`](/ja/reference/configuration/) もここに適用されます: プリストリームの +429 は、翻訳された `openai-chat` / Anthropic リクエスト経路と同様に、同じキーで同一リクエストを +待機して再送します。カスタム `runTurn` トランスポートは HTTP リトライ ループの対象外です。 + - `forward` URL → `{baseUrl}/responses`。`key` provider はデフォルトで従来の `{baseUrl}/v1/responses` 構築を使います。 - `key` provider は検証済みの相対 `responsesPath` を設定できます。adapter は `baseUrl` 末尾の `/` を 1 つ除き、`{trimmedBaseUrl}{responsesPath}` に送信します。Ark Agent Plan では `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` と `responsesPath: "/responses"` を使います。 - `forward` モードでは安全なヘッダー許可リスト(`FORWARD_HEADERS`)だけを中継します。authorization、ChatGPT account id、OpenAI beta/originator/session ヘッダーが対象です。この ChatGPT ログイン経路は [サイドカー](/ja/guides/sidecars/) にも使われます。 diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 968014ce2..2ac59ecfd 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -35,6 +35,10 @@ shipped v1 config는 marker 2의 단일 옵션 행으로 자동 이관됩니다. | `forward` | **수신된 Codex 인증 헤더를** 프로바이더에 그대로 중계합니다 — 키를 저장하지 않습니다. ChatGPT 로그인 패스스루입니다. | OpenAI (`openai-responses` 어댑터). | | `oauth` | 저장된 OAuth 액세스 토큰을 불러와 bearer 키로 사용하며, 만료 전에 자동 갱신합니다. | xAI, Anthropic, Kimi, Kiro, Google Antigravity, Cursor. | +[`retryOn429`](/ko/reference/configuration/)(동일 키 429 재시도)는 API 키 프로바이더 +(`authMode: "key"`)에만 적용됩니다. OAuth·forward·로컬 프리셋은 제외됩니다 — 같은 토큰을 +재전송해서는 안 되며, 로컬 런타임에는 보존할 원격 키가 없습니다. + ## 1. ChatGPT 로그인 (forward / 패스스루) 기본 프로바이더는 **API 키가 필요 없습니다**. 기존 `codex login`의 자격 증명을 OpenAI Responses 백엔드로 diff --git a/docs-site/src/content/docs/ko/reference/adapters.md b/docs-site/src/content/docs/ko/reference/adapters.md index ddfe86864..5cd2834d1 100644 --- a/docs-site/src/content/docs/ko/reference/adapters.md +++ b/docs-site/src/content/docs/ko/reference/adapters.md @@ -45,6 +45,10 @@ interface ProviderAdapter { **변환하지 않은 채** 스트리밍합니다. **인증:** `forward`(호출자 헤더 중계) 또는 `key`. +`key` 인증에서는 [`retryOn429`](/ko/reference/configuration/)도 여기에 적용됩니다: 사전 스트림 +429는 번역된 `openai-chat`/Anthropic 요청 경로와 동일하게 같은 키로 동일 요청을 대기 후 +재전송합니다. 커스텀 `runTurn` 전송은 HTTP 재시도 루프에 포함되지 않습니다. + - `forward` URL → `{baseUrl}/responses`. `key` provider는 기본적으로 기존 `{baseUrl}/v1/responses` 구성을 사용합니다. - `key` provider는 검증된 상대 `responsesPath`를 설정할 수 있습니다. adapter는 `baseUrl` 끝의 `/` 하나를 제거하고 `{trimmedBaseUrl}{responsesPath}`로 전송합니다. Ark Agent Plan은 `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"`와 `responsesPath: "/responses"`를 사용합니다. - `forward` 모드에서는 안전한 헤더 허용 목록(`FORWARD_HEADERS`)만 중계합니다. authorization, diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 25cd3b962..c99d3e737 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -45,6 +45,11 @@ provider — xAI, Kimi, DeepSeek, GLM, Groq, OpenRouter, Ollama (local & cloud), streams the response back **untranslated**. **Auth:** `forward` (relay the caller's headers) or `key`. +For `key` auth, [`retryOn429`](/reference/configuration/) applies here too: a pre-stream 429 +waits and replays the identical request on the same key before any other handling, exactly like +the translated `openai-chat` / Anthropic request path. Custom `runTurn` transports are not part +of the HTTP retry loop. + - `forward` URL → `{baseUrl}/responses`. A `key` provider defaults to the legacy `{baseUrl}/v1/responses` construction. - A `key` provider may set a validated relative `responsesPath`; the adapter removes one trailing slash from `baseUrl` and sends `{trimmedBaseUrl}{responsesPath}`. For Ark Agent Plan, use `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` with `responsesPath: "/responses"`. - In `forward` mode only a safe header allowlist is relayed (`FORWARD_HEADERS`): authorization, diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 8bf6be3bc..36db08dd4 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -41,6 +41,11 @@ description: Все способы, которыми opencodex аутентиф | `forward` | Передаёт провайдеру **входящие заголовки аутентификации Codex** без изменений — ключ не хранится. Это сквозной режим (passthrough) входа через ChatGPT. | OpenAI (адаптер `openai-responses`). | | `oauth` | Берёт сохранённый OAuth-токен доступа (автоматически обновляется до истечения срока) и использует его как bearer-ключ. | xAI, Anthropic, Kimi, Kiro, Google Antigravity, Cursor, GitHub Copilot. | +Повтор при 429 на том же ключе ([`retryOn429`](/ru/reference/configuration/)) применим только к +провайдерам с API-ключом (`authMode: "key"`). Пресеты OAuth, forward и local исключены — их +учётные данные нельзя повторно отправлять по тому же токену, а у локальных сред выполнения нет +удалённого ключа. + ## 1. Вход через ChatGPT (forward / passthrough) Провайдеру `openai` **не нужен API-ключ**. Direct пересылает учётные данные вашего существующего diff --git a/docs-site/src/content/docs/ru/reference/adapters.md b/docs-site/src/content/docs/ru/reference/adapters.md index 80c7ae647..38e479577 100644 --- a/docs-site/src/content/docs/ru/reference/adapters.md +++ b/docs-site/src/content/docs/ru/reference/adapters.md @@ -48,6 +48,10 @@ interface ProviderAdapter { запроса и стримит ответ обратно **без преобразования**. **Аутентификация:** `forward` (ретрансляция заголовков вызывающей стороны) или `key`. +При `key`-аутентификации [`retryOn429`](/ru/reference/configuration/) действует и здесь: 429 до +начала потока ждёт и повторяет идентичный запрос на том же ключе, как и в переводимом пути +`openai-chat`/Anthropic. Пользовательские транспорты `runTurn` в цикл HTTP-повторов не входят. + - URL для `forward` → `{baseUrl}/responses`. Провайдер с `key` по умолчанию сохраняет прежнее построение `{baseUrl}/v1/responses`. - Провайдер с `key` может задать проверенный относительный `responsesPath`: адаптер удаляет один завершающий `/` из `baseUrl` и отправляет запрос на `{trimmedBaseUrl}{responsesPath}`. Для Ark Agent Plan используйте `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` и `responsesPath: "/responses"`. - В режиме `forward` ретранслируется только безопасный allowlist заголовков (`FORWARD_HEADERS`): diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index c86149aa5..353f91027 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -34,6 +34,10 @@ shipped v1 配置自动迁移到 marker 2 的单一选项行。原配置只保 | `forward` | 将**你传入的 Codex 认证请求头**原样转发给提供商——不存储任何密钥。这就是 ChatGPT 登录的透传方式。 | OpenAI(`openai-responses` adapter)。 | | `oauth` | 读取已存储的 OAuth 访问令牌(过期前自动刷新),并将其用作 bearer 密钥。 | xAI、Anthropic、Kimi、Kiro、Google Antigravity、Cursor。 | +[`retryOn429`](/zh-cn/reference/configuration/)(同 key 的 429 重试)仅适用于 API-key 提供商 +(`authMode: "key"`)。OAuth、forward 与本地预设均被排除——同一 token 绝不可重放,本地运行时 +也没有需要保留的远程 key。 + ## 1. ChatGPT 登录(forward / 透传) 默认提供商**不需要 API 密钥**。它将你现有 `codex login` 的凭据直接转发到 OpenAI Responses 后端: diff --git a/docs-site/src/content/docs/zh-cn/reference/adapters.md b/docs-site/src/content/docs/zh-cn/reference/adapters.md index 6a3d89a98..bd611fdd2 100644 --- a/docs-site/src/content/docs/zh-cn/reference/adapters.md +++ b/docs-site/src/content/docs/zh-cn/reference/adapters.md @@ -43,6 +43,10 @@ interface ProviderAdapter { **不经转换**地流式传回。 **认证:** `forward`(转发调用方 header)或 `key`。 +使用 `key` 认证时,[`retryOn429`](/zh-cn/reference/configuration/) 同样适用:流开始前的 429 +会等待并在同一 key 上重放完全相同请求,与翻译后的 `openai-chat`/Anthropic 请求路径一致。 +自定义 `runTurn` 传输不在 HTTP 重试循环之内。 + - `forward` URL → `{baseUrl}/responses`。`key` provider 默认保留原有的 `{baseUrl}/v1/responses` 构造。 - `key` provider 可设置经过验证的相对 `responsesPath`;adapter 会移除 `baseUrl` 末尾的一个 `/`,并向 `{trimmedBaseUrl}{responsesPath}` 发送请求。Ark Agent Plan 使用 `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` 和 `responsesPath: "/responses"`。 - `forward` 模式只会转发安全的 header allowlist(`FORWARD_HEADERS`):authorization、ChatGPT diff --git a/src/config.ts b/src/config.ts index 3474325ec..a7bc9a8f6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1111,6 +1111,45 @@ function warnDegradedStreamMode(rawParsed: unknown, validated: OcxConfig): void } } +/** + * Load-time degradation for `retryOn429` (loadConfig only): one hand-edited invalid optional + * field (e.g. `attempts: 0` or a string) must not trip the whole provider schema and hide every + * provider/key behind a default config. Invalid fields are dropped with a warning; the management + * write boundary still rejects invalid policies explicitly. + */ +function sanitizeRetryOn429ForLoad(parsed: unknown): void { + if (!parsed || typeof parsed !== "object") return; + const root = parsed as Record; + const providers = root.providers; + if (!providers || typeof providers !== "object" || Array.isArray(providers)) return; + for (const [name, provider] of Object.entries(providers as Record)) { + if (!provider || typeof provider !== "object" || Array.isArray(provider)) continue; + const p = provider as Record; + const policy = p.retryOn429; + if (policy === undefined) continue; + if (!policy || typeof policy !== "object" || Array.isArray(policy)) { + delete p.retryOn429; + console.warn(`⚠️ config.json providers.${name}.retryOn429 ${JSON.stringify(policy)} is invalid — ignoring the policy`); + continue; + } + const fields: Array<[string, (value: unknown) => boolean]> = [ + ["enabled", value => typeof value === "boolean"], + ["attempts", value => typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 20], + ["intervalMs", value => typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 600_000], + ["maxIntervalMs", value => typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 600_000], + ["respectRetryAfter", value => typeof value === "boolean"], + ]; + const cleaned: Record = {}; + for (const [key, isValid] of fields) { + const value = (policy as Record)[key]; + if (value === undefined) continue; + if (isValid(value)) cleaned[key] = value; + else console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} ${JSON.stringify(value)} is invalid — ignoring the field`); + } + p.retryOn429 = cleaned; + } +} + /** * Companion to {@link warnDegradedStreamMode} for a blank persisted `hostname`. The bind * falls back to loopback, which is the safe direction but not what the file asked for — @@ -1322,6 +1361,7 @@ export function loadConfig(): OcxConfig { try { const raw = readFileSync(configPath, "utf-8").replace(/^\uFEFF/, ""); const parsed = JSON.parse(raw); + sanitizeRetryOn429ForLoad(parsed); const result = configSchema.safeParse(parsed); if (result.success) { const config = normalizeApiKeyIds(result.data as OcxConfig); diff --git a/src/images/loop.ts b/src/images/loop.ts index 0c020c6ca..ad63ea789 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -236,6 +236,8 @@ export interface ImageBridgeDeps { on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; + /** Telemetry for a same-target 429 replay send (records the `rate-limit-429` recovery kind). */ + onRateLimitRetrySend?: () => void; /** Called when the bridged Responses stream completes (parity with runTurn / routed paths). */ onCompletedResponse?: (response: Record, providerState?: OcxProviderContinuationState) => void; /** WebSocket Responses path only — leave response id empty for protocol compatibility. */ @@ -425,7 +427,7 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { @@ -474,16 +476,23 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } try { await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitRetryPolicy, prepared.response.headers.get("retry-after"), Date.now()), + rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()), signal, ); } catch { - try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } throw new LoopError(499, "client closed request during image-bridge"); } - try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // The deliberate backoff must not consume the cumulative response-header deadline: + // restart it so the replay gets a fresh connect budget (504 stays reserved for real + // upstream latency). + headerDeadline.clear(); + headerDeadline = clearableDeadline(connectTimeoutMs, signal); + deps.onRateLimitRetrySend?.(); // Stall-watchdog seam between bounded retry fetches. yield { type: "heartbeat" }; prepared = await fetchOnce(adapter); diff --git a/src/providers/derive.ts b/src/providers/derive.ts index 03e66f5ca..4c436a6cb 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -108,7 +108,9 @@ export function providerConfigSeed(entry: ProviderRegistryEntry): OcxProviderCon baseUrl: entry.baseUrl, ...(entry.apiKeyTransport !== undefined ? { apiKeyTransport: entry.apiKeyTransport } : {}), ...(entry.responsesPath ? { responsesPath: entry.responsesPath } : {}), - authMode: entry.authKind === "local" ? undefined : entry.authKind, + // Preserve the registry auth kind verbatim (including "local") so fail-closed gates that + // distinguish local runtimes from API-key providers keep working after the seed round-trip. + authMode: entry.authKind, ...(entry.codexAccountMode ? { codexAccountMode: entry.codexAccountMode } : {}), ...(entry.keyOptional !== undefined ? { keyOptional: entry.keyOptional } : {}), ...(entry.freeTier !== undefined ? { freeTier: entry.freeTier } : {}), diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index e6ca014ef..56ad55053 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -49,7 +49,9 @@ function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): const timestamp = Date.parse(text); if (!Number.isFinite(timestamp)) return undefined; const delay = timestamp - now; - return delay > 0 ? Math.min(delay, MAX_COOLDOWN_MS) : undefined; + // A valid HTTP-date whose retry time has already passed is an immediate retry, exactly like + // numeric `Retry-After: 0` — never a malformed-header fallback to the fixed interval. + return Math.min(Math.max(delay, 1), MAX_COOLDOWN_MS); } function isKeyInCooldown(providerName: string, keyId: string, now = Date.now()): boolean { @@ -85,7 +87,11 @@ export function rateLimitRetryPolicyFor( ): Required | null { const policy = provider.retryOn429; if (!policy || policy.enabled === false) return null; - if (provider.authMode === "oauth" || provider.authMode === "forward" || provider.authMode === "local") return null; + // Fail closed: only explicit key auth or the documented omitted-default (undefined == key for + // custom API-key providers) may use same-key replays. OAuth/forward are never replayed on the + // same token, local runtimes have no remote key to preserve, and unknown/custom values are + // rejected rather than guessed at. + if (provider.authMode !== undefined && provider.authMode !== "key") return null; return { enabled: policy.enabled ?? DEFAULT_RATE_LIMIT_RETRY.enabled, attempts: policy.attempts ?? DEFAULT_RATE_LIMIT_RETRY.attempts, diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index b06a00694..5d508d6cf 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1642,21 +1642,24 @@ async function handleResponsesInner( && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; + // Release the unread 429 body before the backoff (only the header is needed for the wait). + const retryAfterHeader = upstreamResponse.headers.get("retry-after"); + cancelResponseBodyBestEffort(upstreamResponse); try { await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitPolicy, upstreamResponse.headers.get("retry-after"), Date.now()), + rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), options.abortSignal, ); } catch { - cancelResponseBodyBestEffort(upstreamResponse); upstream.abort(); return clientCancelledResponse(); } - cancelResponseBodyBestEffort(upstreamResponse); try { upstreamResponse = await fetchWithTransientRetry( recovery => { - noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery); + // The first send of every replay is itself a rate-limit retry; inner transient-5xx + // recoveries keep their own label (recovery is provided for those). + noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery ?? "rate-limit-429"); return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({ method: request.method, headers: request.headers, @@ -2091,6 +2094,7 @@ async function handleResponsesInner( ); }, retryOn429Policy: rateLimitRetryPolicyFor(route.provider), + onRateLimitRetrySend: () => noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, "rate-limit-429"), ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), ...(options.forceEmptyResponseId ? { forceEmptyResponseId: true } : {}), onCompletedResponse: (response, providerState) => @@ -2160,6 +2164,7 @@ async function handleResponsesInner( ); }, retryOn429Policy: rateLimitRetryPolicyFor(route.provider), + onRateLimitRetrySend: () => noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, "rate-limit-429"), }); // Register the sidecar stream as an active turn so drainAndShutdown waits for (or aborts) // in-flight web-search turns instead of skipping them during graceful shutdown. @@ -2436,18 +2441,21 @@ async function handleResponsesInner( && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; + // Release the unread 429 body BEFORE the backoff: only the header matters for the + // wait, and under a 429 storm the sockets would otherwise accumulate for the whole + // configured interval (same pattern as the key-failover branch below). + const retryAfterHeader = upstreamResponse.headers.get("retry-after"); + cancelResponseBodyBestEffort(upstreamResponse); try { await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitPolicy, upstreamResponse.headers.get("retry-after"), Date.now()), + rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), options.abortSignal, ); } catch { - cancelResponseBodyBestEffort(upstreamResponse); cleanupUpstreamAbort(); upstream.abort(); return clientCancelledResponse(); } - cancelResponseBodyBestEffort(upstreamResponse); const result = await rebuildAndRefetch("rate-limit-429"); if ("failed" in result) return result.failed; upstreamResponse = result; @@ -2572,7 +2580,12 @@ async function handleResponsesInner( const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let imageTierBias = 0; let response: Response | undefined; - const fetchContinuation = async (): Promise => { + // Same-target 429 budget is per REQUEST, not per failover iteration: a key/account rotation + // or a 413 tier retry that comes back 429 must not re-arm a fresh budget (parity with the + // main recovery loop above). + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; + const fetchContinuation = async (replay = false): Promise => { const continuationRequest = await activeAdapter.buildRequest(nextParsed, { headers: selectedForwardHeaders, translatorBudget, @@ -2585,7 +2598,7 @@ async function handleResponsesInner( if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; try { if (activeAdapter.fetchResponse) { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate); + noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replay ? "rate-limit-429" : undefined); return await activeAdapter.fetchResponse(continuationRequest, { abortSignal: upstream.signal, timeoutMs: connectMs, @@ -2594,7 +2607,7 @@ async function handleResponsesInner( } return await fetchWithResetRetry( recovery => { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery); + noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery ?? (replay ? "rate-limit-429" : undefined)); return fetchWithHeaderTimeout( continuationRequest.url, applyUpstreamRecoveryInit({ @@ -2629,31 +2642,33 @@ async function handleResponsesInner( // Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover: // a primary-key rate-limit blip replays on the SAME key, matching the main recovery // loop; only after the attempts are exhausted does the continuation fail over. - const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); - let rateLimitRetries = 0; while ( response.status === 429 && rateLimitPolicy !== null && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; + // Release the unread 429 body before the backoff (only the header is needed for the wait). + const retryAfterHeader = response.headers.get("retry-after"); + try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitPolicy, response.headers.get("retry-after"), Date.now()), - options.abortSignal, + rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), + // Listen on the upstream signal: once the SSE body is being streamed, a client + // cancel aborts `upstream` through the bridge, and upstream is also linked from + // options.abortSignal — so this covers both cancellation paths. + upstream.signal, ); } catch { - try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } - if (options.abortSignal?.aborted) { + if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; } else { yield { type: "error", message: "Provider continuation failed: retry wait interrupted" }; } return; } - try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { - response = await fetchContinuation(); + response = await fetchContinuation(true); } catch (error) { if (options.abortSignal?.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index b220fbf0e..d94a664e7 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -247,6 +247,8 @@ export interface WebSearchLoopDeps { on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; + /** Telemetry for a same-target 429 replay send (records the `rate-limit-429` recovery kind). */ + onRateLimitRetrySend?: () => void; } /** @@ -316,7 +318,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { @@ -368,16 +370,23 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } try { await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitRetryPolicy, prepared.response.headers.get("retry-after"), Date.now()), + rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()), signal, ); } catch { - try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } throw new LoopError(499, "client closed request during web-search"); } - try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // The deliberate backoff must not consume the cumulative response-header deadline: + // restart it so the replay gets a fresh connect budget (504 stays reserved for real + // upstream latency). + headerDeadline.clear(); + headerDeadline = clearableDeadline(connectTimeoutMs, signal); + deps.onRateLimitRetrySend?.(); // Stall-watchdog seam between bounded retry fetches. yield { type: "heartbeat" }; prepared = await fetchOnce(adapter); diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index d0142ab84..601ce28de 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -232,11 +232,14 @@ failover). Codex never retries 429 client-side (openai/codex#30471), so this is defense for those providers; the final 429 still carries `Retry-After` for clients that honor it. Concurrent requests each honor their own policy — there is no process-wide shared cooldown (unlike the Kiro pattern), so a rate-limit storm multiplies upstream volume by at most -`attempts + 1` per request. The wait is abort-aware: once the server observes the client -disconnect (Bun propagates it asynchronously, observed 1–10 s), the sleep is interrupted, the -unread 429 body is released, and the request is cancelled with 499 before any replay; because -the propagation is async, a replay may precede the cancel if the interval elapses first -(bounded by the same `attempts` budget). +`attempts + poolKeys` per request (same-key replays, then failover keys). Every surface +releases the unread 429 body before the backoff, records the `rate-limit-429` recovery kind on +replay sends, and the bridge loops restart their response-header deadline after each deliberate +wait so backoffs never consume the connect budget or surface as a 504. The wait is abort-aware: +once the server observes the client disconnect (Bun propagates it asynchronously, observed +1–10 s), the sleep is interrupted, the unread 429 body is released, and the request is +cancelled with 499 before any replay; because the propagation is async, a replay may precede +the cancel if the interval elapses first (bounded by the same `attempts` budget). [Decision Log] - 목적과 의도: Prevent Kiro progress from becoming a false final answer, reject invalid empty completion retries, and stop concurrent transient 429s from consuming independent retry budgets. diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 35d16d202..65cc01b10 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -109,6 +109,41 @@ test("an unrelated loadConfig does not refresh the armed baseline", () => { expect((diskConfig().claudeCode as Record).authMode).toBe("proxy"); }); +test("an invalid retryOn429 field degrades at load instead of discarding the config", () => { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: { attempts: 0, intervalMs: 120, respectRetryAfter: false }, + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test).toBeDefined(); + // Invalid field dropped with a warning; valid fields kept; missing fields defaulted. + expect(live.providers.test.retryOn429).toEqual({ intervalMs: 120, respectRetryAfter: false }); +}); + +test("a non-object retryOn429 degrades at load instead of discarding the config", () => { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: "enabled", + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test).toBeDefined(); + expect(live.providers.test.retryOn429).toBeUndefined(); +}); + // R4-1: the request path. A 429 mid-turn rotates a key and saves, with no user action. test("a 429 key rotation does not clobber the hand edit", async () => { const { rotateKeyOn429 } = await import("../src/providers/key-failover"); diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index 132ef0d4f..e6f3fc407 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -180,6 +180,7 @@ describe("runWithImageBridge", () => { test("retryOn429 replays on the same key before on429 rotation", async () => { let sends = 0; let rotations = 0; + let retrySends = 0; const retryingAdapter: ProviderAdapter = { ...mockAdapter, fetchResponse: async () => { @@ -203,11 +204,15 @@ describe("runWithImageBridge", () => { rotations += 1; return null; }, + onRateLimitRetrySend: () => { + retrySends += 1; + }, }); const sse = await response.text(); expect(sse).toContain("recovered"); expect(sends).toBe(2); expect(rotations).toBe(0); + expect(retrySends).toBe(1); }); test("forced-final clears named image tool_choice", async () => { diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 3ed777716..7f2047416 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -209,6 +209,15 @@ describe("provider registry parity", () => { } }); + test("providerConfigSeed preserves the registry auth kind, including local", () => { + const local = PROVIDER_REGISTRY.find(entry => entry.authKind === "local"); + expect(local).toBeDefined(); + expect(providerConfigSeed(local!).authMode).toBe("local"); + const key = PROVIDER_REGISTRY.find(entry => entry.id === "deepseek"); + expect(key).toBeDefined(); + expect(providerConfigSeed(key!).authMode).toBe("key"); + }); + test("CN provider defaults and context windows match the audited registry refresh", () => { const deepseek = PROVIDER_REGISTRY.find(entry => entry.id === "deepseek"); expect(deepseek).toMatchObject({ diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts index 75e77dba3..256bdb65e 100644 --- a/tests/rate-limit-retry.test.ts +++ b/tests/rate-limit-retry.test.ts @@ -12,7 +12,7 @@ describe("rateLimitRetryPolicyFor", () => { expect(rateLimitRetryPolicyFor({ retryOn429: { enabled: false } } as OcxProviderConfig)).toBeNull(); }); - test("null for OAuth, forward, and local providers (no remote key to preserve)", () => { + test("null for OAuth, forward, local, and unknown auth modes (fail closed)", () => { expect(rateLimitRetryPolicyFor({ authMode: "oauth", retryOn429: {}, @@ -25,6 +25,10 @@ describe("rateLimitRetryPolicyFor", () => { authMode: "local", retryOn429: {}, } as OcxProviderConfig)).toBeNull(); + expect(rateLimitRetryPolicyFor({ + authMode: "custom-unknown", + retryOn429: {}, + } as OcxProviderConfig)).toBeNull(); expect(rateLimitRetryPolicyFor({ authMode: "key", retryOn429: {}, @@ -72,6 +76,12 @@ describe("rateLimitRetryDelayMs", () => { expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:28:00 GMT", now)).toBe(30_000); }); + test("an already-expired HTTP-date Retry-After retries immediately", () => { + const now = Date.parse("2026-10-21T07:28:00Z"); + expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:27:30 GMT", now)).toBe(1); + expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:28:00 GMT", now)).toBe(1); + }); + test("Retry-After 0 retries immediately instead of falling back to the interval", () => { expect(rateLimitRetryDelayMs(policy, "0", 1_000_000)).toBe(1); }); diff --git a/tests/server-rate-limit-retry-e2e.test.ts b/tests/server-rate-limit-retry-e2e.test.ts index 1811e6148..9ecb7db76 100644 --- a/tests/server-rate-limit-retry-e2e.test.ts +++ b/tests/server-rate-limit-retry-e2e.test.ts @@ -238,12 +238,14 @@ describe("server same-target 429 retry (end-to-end)", () => { const originalFetch = globalThis.fetch; let sends = 0; const seenAuth: string[] = []; + const seenBodies: string[] = []; globalThis.fetch = (async (input, init) => { const url = input instanceof Request ? input.url : String(input); if (url === "https://passthrough.test/v1/responses") { sends += 1; const auth = new Headers(init?.headers).get("authorization") ?? ""; seenAuth.push(auth); + seenBodies.push(String(init?.body)); if (sends === 1) { return new Response(JSON.stringify({ error: { message: "rate limited" } }), { status: 429, @@ -287,6 +289,56 @@ describe("server same-target 429 retry (end-to-end)", () => { "Bearer key-alpha-000111222333", "Bearer key-alpha-000111222333", ]); + expect(seenBodies).toHaveLength(2); + expect(seenBodies[0]).toBe(seenBodies[1]); + } finally { + server?.stop(true); + globalThis.fetch = originalFetch; + } + }); + + test("retry budget stays per request across multi-key failover (never re-arms)", async () => { + const originalFetch = globalThis.fetch; + let sends = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + sends += 1; + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return originalFetch(input, init); + }) as typeof fetch; + + let server: ReturnType | null = null; + try { + const config = { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + apiKeyPool: [ + { id: "k1", key: "key-alpha-000111222333", addedAt: 1 }, + { id: "k2", key: "key-beta-444555666777", addedAt: 2 }, + ], + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + saveConfig(config); + server = startServer(0); + const res = await postResponses(server.url, "blsc/DeepSeek-V4-Flash"); + expect(res.status).toBe(429); + // attempts(1) on the first key + one failover key = 3 sends; a re-armed budget would + // have replayed on the second key too (4+ sends). + expect(sends).toBe(3); } finally { server?.stop(true); globalThis.fetch = originalFetch; diff --git a/tests/terminal-guard-server.test.ts b/tests/terminal-guard-server.test.ts index fe246fd0a..8c88007a2 100644 --- a/tests/terminal-guard-server.test.ts +++ b/tests/terminal-guard-server.test.ts @@ -91,8 +91,10 @@ describe("server terminal guard integration", () => { }, } as unknown as OcxConfig; let sends = 0; + const requestBodies: string[] = []; globalThis.fetch = (async (_input, init) => { sends += 1; + requestBodies.push(String(init?.body ?? "")); if (sends === 2) { return new Response(JSON.stringify({ error: { message: "rate limited" } }), { status: 429, @@ -119,6 +121,54 @@ describe("server terminal guard integration", () => { expect(sends).toBe(3); expect(text).toContain("response.completed"); expect(text).toContain("exec_command"); + // The 429 continuation (send 2) and its same-key replay (send 3) must be byte-identical. + expect(requestBodies).toHaveLength(3); + expect(requestBodies[1]).toBe(requestBodies[2]); + }); + + test("terminal-guard continuation retry budget stays per request across key failover", async () => { + const budgetConfig = { + ...config, + providers: { + "claude-se": { + adapter: "anthropic", + baseUrl: "https://example.test", + apiKey: "sk-test", + apiKeyPool: [ + { id: "k1", key: "sk-test", addedAt: 1 }, + { id: "k2", key: "sk-test-2", addedAt: 2 }, + ], + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as unknown as OcxConfig; + let sends = 0; + globalThis.fetch = (async (_input, init) => { + sends += 1; + if (sends === 1) { + return anthropicSse(firstTurn); + } + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "se-claude-opus-4.8", + input: "请检查这个问题并修复代码", + stream: true, + tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }], + }), + }), budgetConfig, { model: "", provider: "" }); + + await response.text(); + // initial turn + continuation retry (same key) + failover continuation (second key) = 4. + // A per-iteration budget would replay on the second key too (5+ sends). + expect(sends).toBe(4); }); }); diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index 702bdbc1b..510df4638 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -17,6 +17,7 @@ import { usageTotalTokens, usageReadCacheStatsForTests, usageLogRevisionKey, + type PersistedUsageEntry, } from "../src/usage/log"; let testDir = ""; @@ -37,7 +38,7 @@ afterEach(() => { describe("usage log", () => { test("persists the rate-limit-429 recovery kind on attempts", () => { - appendUsageEntry({ + const entry: PersistedUsageEntry = { requestId: "ocx-ratelimit-kind", timestamp: 1, provider: "blsc", @@ -56,7 +57,8 @@ describe("usage log", () => { recoveryKinds: ["rate-limit-429", "rate-limit-429"], usageStatus: "reported", }], - } as never); + }; + appendUsageEntry(entry); expect(readUsageEntries()[0]?.attempts?.[0]?.recoveryKinds).toEqual(["rate-limit-429"]); }); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index bbed3778f..eada67f78 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -631,6 +631,7 @@ describe("web-search sidecar native web_search_call emission", () => { let sends = 0; let rotations = 0; + let retrySends = 0; const retryingAdapter: ProviderAdapter = { name: "mock-retry429", buildRequest: () => ({ @@ -666,6 +667,9 @@ describe("web-search sidecar native web_search_call emission", () => { rotations += 1; return null; }, + onRateLimitRetrySend: () => { + retrySends += 1; + }, }); expect(response.status).toBe(200); const frames = await collectSse(response.body!); @@ -674,6 +678,7 @@ describe("web-search sidecar native web_search_call emission", () => { expect(output.find(o => o.type === "message")?.content?.[0]?.text).toBe("answer after same-key retry"); expect(sends).toBe(2); expect(rotations).toBe(0); + expect(retrySends).toBe(1); }); test("loop 429 with exhausted pool (on429 null) surfaces the provider error", async () => { From 0feede1a104881dd14c7d7cfc6f54d0c35f52b3c Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 03:19:57 +0800 Subject: [PATCH 05/29] docs: add JSDoc for retryOn429 helpers (CodeRabbit docstring coverage) --- src/providers/key-failover.ts | 5 +++++ src/server/responses/core.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 56ad55053..6e6a994b1 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -37,6 +37,11 @@ function cooldownKey(providerName: string, keyId: string): string { return `${providerName}\0${keyId}`; } +/** + * Parse an upstream `Retry-After` header: numeric seconds (including `0`) or an HTTP-date. + * Returns a bounded delay in ms (1..MAX_COOLDOWN_MS), or undefined when the value is + * malformed. An HTTP-date already in the past yields an immediate (1 ms) retry. + */ function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined { const text = value?.trim(); if (!text) return undefined; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 5d508d6cf..a4a485bb4 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2585,6 +2585,11 @@ async function handleResponsesInner( // main recovery loop above). const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); let rateLimitRetries = 0; + /** + * Build and fetch one terminal-guard continuation. `replay` marks a same-target 429 + * replay so its send records the `rate-limit-429` recovery kind; the adapter rebuild is + * deterministic for the same parsed request (tests assert byte-identical replays). + */ const fetchContinuation = async (replay = false): Promise => { const continuationRequest = await activeAdapter.buildRequest(nextParsed, { headers: selectedForwardHeaders, From 568e565c1bb21f683a195d05769296b50e287c0b Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 03:42:26 +0800 Subject: [PATCH 06/29] fix(proxy): second review-bot pass (awaited body cancel, stale-deadline race, config warnings, locale docs) - images/web-search loops: AWAIT the unread 429 body cancellation before the backoff; clear the old header deadline BEFORE the sleep; re-check client cancellation after the wait so 499 wins over stale-deadline edges; start the fresh deadline before telemetry and replay - config: sanitizeRetryOn429ForLoad warns about misnamed keys (e.g. attempt) instead of silently dropping them - docs: locale adapter pages state same-key replay runs before other handling/failover; provider guide (English + ja/ko/ru/zh-cn) states retryOn429 is opt-in (absent = off); devlog and structure/04 note the awaited cancellation and the 499-before-replay guarantee - test: config-user-edits covers the misnamed-key drop --- .../260802_429_same_target_retry/010_design.md | 14 +++++++++----- docs-site/src/content/docs/guides/providers.md | 3 ++- docs-site/src/content/docs/ja/guides/providers.md | 1 + .../src/content/docs/ja/reference/adapters.md | 5 +++-- docs-site/src/content/docs/ko/guides/providers.md | 3 ++- .../src/content/docs/ko/reference/adapters.md | 5 +++-- docs-site/src/content/docs/ru/guides/providers.md | 3 ++- .../src/content/docs/ru/reference/adapters.md | 5 +++-- .../src/content/docs/zh-cn/guides/providers.md | 2 +- .../src/content/docs/zh-cn/reference/adapters.md | 4 ++-- src/config.ts | 6 ++++++ src/images/loop.ts | 11 ++++++++--- src/web-search/loop.ts | 11 ++++++++--- structure/04_transports-and-sidecars.md | 8 +++++--- tests/config-user-edits.test.ts | 5 +++-- 15 files changed, 58 insertions(+), 28 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index 3a13fe481..d34800c01 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -28,7 +28,8 @@ existing multi-key failover runs. Default off → zero behavior change for exist dropped, never a config-rejecting error; the outer provider schema stays passthrough). Load-time degradation: one hand-edited invalid optional field (e.g. `attempts: 0`) is dropped with a warning instead of tripping the whole schema and hiding all providers behind a default - config; the management write boundary still rejects invalid policies. + config; misnamed keys (e.g. `attempt`) are warned about too; the management write boundary + still rejects invalid policies. - `src/providers/key-failover.ts`: `rateLimitRetryPolicyFor` (normalize/default) and `rateLimitRetryDelayMs` (Retry-After seconds/HTTP-date → capped, else `intervalMs`), reusing the existing `parseRetryAfterMs` cooldown parser; the fixed fallback is capped at @@ -50,9 +51,10 @@ existing multi-key failover runs. Default off → zero behavior change for exist - image/video bridge and web-search sidecar loops (`src/images/loop.ts`, `src/web-search/loop.ts`) — before their `on429` key rotation; - Anthropic terminal-guard continuations — before key/account failover. - Every surface releases the unread 429 body BEFORE the backoff, records the - `rate-limit-429` recovery kind on replay sends, and (bridges) restarts the response-header - deadline after each deliberate wait. + Every surface releases (and awaits the cancellation of) the unread 429 body BEFORE the + backoff, records the `rate-limit-429` recovery kind on replay sends, and (bridges) clears the + old response-header deadline before the wait and starts a fresh one afterward, re-checking + client cancellation before telemetry and replay. Covers Responses, chat completions, and routed Claude messages (they all enter `handleResponses`). @@ -84,7 +86,9 @@ existing multi-key failover runs. Default off → zero behavior change for exist `attempts + 1`. - Header deadlines: the image/video and web-search bridge loops restart their response-header deadline after each deliberate wait, so backoffs never consume the connect budget and a - rate-limit wait is never misattributed as a 504 header timeout. + rate-limit wait is never misattributed as a 504 header timeout. The old deadline is cleared + BEFORE the sleep and client cancellation is re-checked after it, so 499 always wins over a + stale-deadline edge. - Expired `Retry-After`: a valid HTTP-date already in the past retries immediately (same as numeric `Retry-After: 0`) instead of falling back to the fixed interval. - Recovery observability: every retry surface records the `rate-limit-429` recovery kind diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index cc2b094bc..29b76262d 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -42,7 +42,8 @@ labels local presets separately; those normally omit both `authMode` and `apiKey The [`retryOn429`](/reference/configuration/) same-key 429 replay applies only to API-key providers (`authMode: "key"`). OAuth, forward, and local presets are excluded — their credentials must never be replayed on the same token, and local runtimes have no remote key to -preserve. +preserve. It is opt-in: when the option is absent the feature is off; object presence enables +it unless `enabled: false`. ## 1. ChatGPT login (forward / passthrough) diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index bbc9dba31..abd66a374 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -38,6 +38,7 @@ max input 922,000 で `*-pro` virtual ID は公開状態を維持し、wire で [`retryOn429`](/ja/reference/configuration/)(同一キーでの 429 リトライ)は API キー プロバイダー (`authMode: "key"`)のみに適用されます。OAuth・forward・ローカル プリセットは除外されます — 同じトークンを再送すべきではなく、ローカルランタイムには保存すべきリモートキーがありません。 +オプトインです: オプションが無ければ無効、オブジェクトがあれば `enabled: false` でない限り有効です。 ## 1. ChatGPT ログイン(forward / パススルー) diff --git a/docs-site/src/content/docs/ja/reference/adapters.md b/docs-site/src/content/docs/ja/reference/adapters.md index 4276d9b62..1b363807d 100644 --- a/docs-site/src/content/docs/ja/reference/adapters.md +++ b/docs-site/src/content/docs/ja/reference/adapters.md @@ -39,8 +39,9 @@ interface ProviderAdapter { **認証:** `forward`(呼び出し元ヘッダー中継)または `key`。 `key` 認証では、[`retryOn429`](/ja/reference/configuration/) もここに適用されます: プリストリームの -429 は、翻訳された `openai-chat` / Anthropic リクエスト経路と同様に、同じキーで同一リクエストを -待機して再送します。カスタム `runTurn` トランスポートは HTTP リトライ ループの対象外です。 +429 は、翻訳された `openai-chat` / Anthropic リクエスト経路と同様に、他の処理やフェイルオーバーに +先立って、同じキーで同一リクエストを待機して再送します。カスタム `runTurn` トランスポートは +HTTP リトライ ループの対象外です。 - `forward` URL → `{baseUrl}/responses`。`key` provider はデフォルトで従来の `{baseUrl}/v1/responses` 構築を使います。 - `key` provider は検証済みの相対 `responsesPath` を設定できます。adapter は `baseUrl` 末尾の `/` を 1 つ除き、`{trimmedBaseUrl}{responsesPath}` に送信します。Ark Agent Plan では `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` と `responsesPath: "/responses"` を使います。 diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 2ac59ecfd..9db6cd455 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -37,7 +37,8 @@ shipped v1 config는 marker 2의 단일 옵션 행으로 자동 이관됩니다. [`retryOn429`](/ko/reference/configuration/)(동일 키 429 재시도)는 API 키 프로바이더 (`authMode: "key"`)에만 적용됩니다. OAuth·forward·로컬 프리셋은 제외됩니다 — 같은 토큰을 -재전송해서는 안 되며, 로컬 런타임에는 보존할 원격 키가 없습니다. +재전송해서는 안 되며, 로컬 런타임에는 보존할 원격 키가 없습니다. 옵트인입니다: 옵션이 없으면 +꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. ## 1. ChatGPT 로그인 (forward / 패스스루) diff --git a/docs-site/src/content/docs/ko/reference/adapters.md b/docs-site/src/content/docs/ko/reference/adapters.md index 5cd2834d1..7816aa325 100644 --- a/docs-site/src/content/docs/ko/reference/adapters.md +++ b/docs-site/src/content/docs/ko/reference/adapters.md @@ -46,8 +46,9 @@ interface ProviderAdapter { **인증:** `forward`(호출자 헤더 중계) 또는 `key`. `key` 인증에서는 [`retryOn429`](/ko/reference/configuration/)도 여기에 적용됩니다: 사전 스트림 -429는 번역된 `openai-chat`/Anthropic 요청 경로와 동일하게 같은 키로 동일 요청을 대기 후 -재전송합니다. 커스텀 `runTurn` 전송은 HTTP 재시도 루프에 포함되지 않습니다. +429는 번역된 `openai-chat`/Anthropic 요청 경로와 동일하게 다른 처리나 페일오버보다 먼저 +같은 키로 동일 요청을 대기 후 재전송합니다. 커스텀 `runTurn` 전송은 HTTP 재시도 루프에 +포함되지 않습니다. - `forward` URL → `{baseUrl}/responses`. `key` provider는 기본적으로 기존 `{baseUrl}/v1/responses` 구성을 사용합니다. - `key` provider는 검증된 상대 `responsesPath`를 설정할 수 있습니다. adapter는 `baseUrl` 끝의 `/` 하나를 제거하고 `{trimmedBaseUrl}{responsesPath}`로 전송합니다. Ark Agent Plan은 `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"`와 `responsesPath: "/responses"`를 사용합니다. diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 36db08dd4..80a8e3268 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -44,7 +44,8 @@ description: Все способы, которыми opencodex аутентиф Повтор при 429 на том же ключе ([`retryOn429`](/ru/reference/configuration/)) применим только к провайдерам с API-ключом (`authMode: "key"`). Пресеты OAuth, forward и local исключены — их учётные данные нельзя повторно отправлять по тому же токену, а у локальных сред выполнения нет -удалённого ключа. +удалённого ключа. Это opt-in: при отсутствии опции функция выключена; наличие объекта включает +её, если только `enabled: false`. ## 1. Вход через ChatGPT (forward / passthrough) diff --git a/docs-site/src/content/docs/ru/reference/adapters.md b/docs-site/src/content/docs/ru/reference/adapters.md index 38e479577..3cac09968 100644 --- a/docs-site/src/content/docs/ru/reference/adapters.md +++ b/docs-site/src/content/docs/ru/reference/adapters.md @@ -49,8 +49,9 @@ interface ProviderAdapter { **Аутентификация:** `forward` (ретрансляция заголовков вызывающей стороны) или `key`. При `key`-аутентификации [`retryOn429`](/ru/reference/configuration/) действует и здесь: 429 до -начала потока ждёт и повторяет идентичный запрос на том же ключе, как и в переводимом пути -`openai-chat`/Anthropic. Пользовательские транспорты `runTurn` в цикл HTTP-повторов не входят. +начала потока ждёт и, до любой другой обработки или фейловера, повторяет идентичный запрос на +том же ключе, как и в переводимом пути `openai-chat`/Anthropic. Пользовательские транспорты +`runTurn` в цикл HTTP-повторов не входят. - URL для `forward` → `{baseUrl}/responses`. Провайдер с `key` по умолчанию сохраняет прежнее построение `{baseUrl}/v1/responses`. - Провайдер с `key` может задать проверенный относительный `responsesPath`: адаптер удаляет один завершающий `/` из `baseUrl` и отправляет запрос на `{trimmedBaseUrl}{responsesPath}`. Для Ark Agent Plan используйте `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` и `responsesPath: "/responses"`. diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index 353f91027..55d002b08 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -36,7 +36,7 @@ shipped v1 配置自动迁移到 marker 2 的单一选项行。原配置只保 [`retryOn429`](/zh-cn/reference/configuration/)(同 key 的 429 重试)仅适用于 API-key 提供商 (`authMode: "key"`)。OAuth、forward 与本地预设均被排除——同一 token 绝不可重放,本地运行时 -也没有需要保留的远程 key。 +也没有需要保留的远程 key。仅在配置后启用,默认关闭;配置了对象即启用,除非 `enabled: false`。 ## 1. ChatGPT 登录(forward / 透传) diff --git a/docs-site/src/content/docs/zh-cn/reference/adapters.md b/docs-site/src/content/docs/zh-cn/reference/adapters.md index bd611fdd2..9d44f3697 100644 --- a/docs-site/src/content/docs/zh-cn/reference/adapters.md +++ b/docs-site/src/content/docs/zh-cn/reference/adapters.md @@ -44,8 +44,8 @@ interface ProviderAdapter { **认证:** `forward`(转发调用方 header)或 `key`。 使用 `key` 认证时,[`retryOn429`](/zh-cn/reference/configuration/) 同样适用:流开始前的 429 -会等待并在同一 key 上重放完全相同请求,与翻译后的 `openai-chat`/Anthropic 请求路径一致。 -自定义 `runTurn` 传输不在 HTTP 重试循环之内。 +会等待并先于其他处理或故障转移,在相同 key 上重放完全相同请求,与翻译后的 +`openai-chat`/Anthropic 请求路径一致。自定义 `runTurn` 传输不在 HTTP 重试循环之内。 - `forward` URL → `{baseUrl}/responses`。`key` provider 默认保留原有的 `{baseUrl}/v1/responses` 构造。 - `key` provider 可设置经过验证的相对 `responsesPath`;adapter 会移除 `baseUrl` 末尾的一个 `/`,并向 `{trimmedBaseUrl}{responsesPath}` 发送请求。Ark Agent Plan 使用 `baseUrl: "https://ark.cn-beijing.volces.com/api/plan/v3"` 和 `responsesPath: "/responses"`。 diff --git a/src/config.ts b/src/config.ts index a7bc9a8f6..677924541 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1146,6 +1146,12 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { if (isValid(value)) cleaned[key] = value; else console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} ${JSON.stringify(value)} is invalid — ignoring the field`); } + const knownKeys = new Set(fields.map(([key]) => key)); + for (const key of Object.keys(policy as Record)) { + if (!knownKeys.has(key)) { + console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} is not a recognized field — ignoring it`); + } + } p.retryOn429 = cleaned; } } diff --git a/src/images/loop.ts b/src/images/loop.ts index ad63ea789..ff971620f 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -478,7 +478,11 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } + // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. + try { await prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // The old header deadline must not stay armed across the deliberate wait: clear it + // before sleeping so a stale expiry can never race the client-cancel path. + headerDeadline.clear(); try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()), @@ -487,10 +491,11 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } + // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. + try { await prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // The old header deadline must not stay armed across the deliberate wait: clear it + // before sleeping so a stale expiry can never race the client-cancel path. + headerDeadline.clear(); try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()), @@ -381,10 +385,11 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise Date: Sun, 2 Aug 2026 03:49:56 +0800 Subject: [PATCH 07/29] docs: lift docstring coverage on the retryOn429 diff --- src/images/loop.ts | 5 +++++ src/providers/derive.ts | 5 +++++ src/server/responses/core.ts | 27 +++++++++++++++++++---- src/web-search/loop.ts | 5 +++++ tests/config-user-edits.test.ts | 2 ++ tests/images/loop.test.ts | 1 + tests/server-rate-limit-retry-e2e.test.ts | 1 + tests/terminal-guard-server.test.ts | 1 + tests/web-search.test.ts | 1 + 9 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/images/loop.ts b/src/images/loop.ts index ff971620f..6f32ca89e 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -339,6 +339,11 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise { const iterParsed: OcxParsedRequest = { ...parsed, diff --git a/src/providers/derive.ts b/src/providers/derive.ts index 4c436a6cb..1d45028e2 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -102,6 +102,11 @@ function cloneNestedRecord(input: Record>): Recor return Object.fromEntries(Object.entries(input).map(([key, value]) => [key, { ...value }])); } +/** + * Build the provider config a registry entry contributes when a preset is materialized. + * The registry auth kind is preserved verbatim (including `"local"`) so fail-closed gates + * keep distinguishing local runtimes from API-key providers after the seed round-trip. + */ export function providerConfigSeed(entry: ProviderRegistryEntry): OcxProviderConfig { return { adapter: entry.adapter, diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index a4a485bb4..0a9ba4754 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -560,6 +560,10 @@ export interface HandleResponsesOptions { +/** + * Build the 499 JSON error the proxy returns when the client disconnects before the + * response completes (`client_cancelled`). + */ export function clientCancelledResponse(): Response { return formatErrorResponse(499, "client_cancelled", "Client cancelled request"); } @@ -1154,6 +1158,10 @@ function finalizeOwnedTranslatorBudget(response: Response, budget: TranslatorBud return finalizedResponse; } +/** + * Route one `/v1/responses` request through the adapter pipeline: recovery loop, passthrough + * wire, image/web-search bridges, and the terminal-guard continuation. + */ export async function handleResponses( req: Request, config: OcxConfig, @@ -1171,6 +1179,10 @@ export async function handleResponses( } } +/** + * Inner implementation of `handleResponses`; owns the pre-stream recovery loop and the + * per-request same-target 429 retry budgets. + */ async function handleResponsesInner( req: Request, config: OcxConfig, @@ -2356,6 +2368,11 @@ async function handleResponsesInner( // comes back 429 cannot silently re-arm a fresh budget (bounded to `attempts` per request). const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); let rateLimitRetries = 0; + /** + * Rebuild the request from the current parsed input (and any image-tier bias) and refetch + * it once, tagging the attempt with the given recovery kind. Rebuilds are deterministic + * for the same parsed request, so same-target replays stay byte-identical. + */ const rebuildAndRefetch = async ( recovery: AttemptRecoveryKind, ): Promise => { @@ -2572,10 +2589,12 @@ async function handleResponsesInner( cancelBodyOnAbort(upstreamResponse.body, upstream.signal); - // Claude can return a clean end_turn after announcing an edit without emitting any tool call. - // Keep the normal request/recovery path above intact, and use this bounded callback only for the - // one internal continuation pass. A continuation failure becomes an in-stream adapter error so - // the client never sees a second hidden HTTP response or an unbounded retry loop. + /** + * One bounded internal re-ask for Anthropic end_turn-without-tool-call turns. Replays the + * continuation on a 429 with the same-key retry budget (hoisted per request), then falls + * back to key/account failover; a failure becomes an in-stream adapter error so the client + * never sees a second hidden HTTP response or an unbounded retry loop. + */ const terminalGuardEnabled = activeAdapter.name === "anthropic" && !options.comboAttempt && !routedCompaction; const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let imageTierBias = 0; diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 3267e7dac..42ad9f197 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -302,6 +302,11 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { // On the forced-answer pass the synthetic web_search tool is gone, so the model MUST answer // from the results already in `messages`. A weak model can still produce a thin answer that diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 11805a419..0f6cff2fa 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -21,11 +21,13 @@ import type { OcxConfig } from "../src/types"; let home: string; let previousHome: string | undefined; +/** Merge a patch into the on-disk config.json, simulating a user hand-edit. */ function writeDiskConfig(patch: Record): void { const current = JSON.parse(readFileSync(getConfigPath(), "utf8")) as Record; writeFileSync(getConfigPath(), JSON.stringify({ ...current, ...patch }, null, 2) + "\n"); } +/** Read the current on-disk config.json as a plain record. */ function diskConfig(): Record { return JSON.parse(readFileSync(getConfigPath(), "utf8")) as Record; } diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index e6f3fc407..0e93d4031 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -95,6 +95,7 @@ const imageCallEvents: AdapterEvent[] = [ { type: "done" }, ]; +/** Run the image bridge with the given per-iteration event streams and return the client-facing SSE text. */ async function runAndGetSSE(streams: AdapterEvent[][], fulfill?: ImageCallResult): Promise { streamQueue = streams.map(s => [...s]); if (fulfill) fulfillResult = fulfill; diff --git a/tests/server-rate-limit-retry-e2e.test.ts b/tests/server-rate-limit-retry-e2e.test.ts index 9ecb7db76..f64b80dfe 100644 --- a/tests/server-rate-limit-retry-e2e.test.ts +++ b/tests/server-rate-limit-retry-e2e.test.ts @@ -36,6 +36,7 @@ const okChatCompletion = JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2, total_tokens: 5 }, }); +/** POST a non-streaming `/v1/responses` request to the proxy under test. */ async function postResponses(serverUrl: string, model: string): Promise { return fetch(new URL("/v1/responses", serverUrl), { method: "POST", diff --git a/tests/terminal-guard-server.test.ts b/tests/terminal-guard-server.test.ts index 8c88007a2..e52eb1d9b 100644 --- a/tests/terminal-guard-server.test.ts +++ b/tests/terminal-guard-server.test.ts @@ -14,6 +14,7 @@ const config = { }, } as unknown as OcxConfig; +/** Build an Anthropic SSE response from raw frames. */ function anthropicSse(body: string): Response { return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }); } diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index eada67f78..741987b66 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -11,6 +11,7 @@ import type { OcxMessage, OcxParsedRequest } from "../src/types"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; import { createTestTranslatorBudget } from "./helpers/translator-budget"; +/** Run the web-search loop with a default test translator budget. */ function runWithWebSearch( deps: Omit & { incomingMeta?: WebSearchLoopDeps["incomingMeta"] }, ): Promise { From 1af83c39cb2a88672cfa0d1ea0d5027ab34dfc10 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 03:55:51 +0800 Subject: [PATCH 08/29] docs: attach JSDoc directly to remaining diff-touched declarations --- src/images/loop.ts | 4 ++++ src/providers/key-failover.ts | 4 ++++ src/server/responses/core.ts | 4 +++- src/types.ts | 4 ++++ src/web-search/loop.ts | 4 ++++ tests/usage-log.test.ts | 1 + 6 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/images/loop.ts b/src/images/loop.ts index 6f32ca89e..0aceb39d5 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -434,6 +434,10 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(), diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 6e6a994b1..2f33cb49e 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -59,6 +59,10 @@ function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): return Math.min(Math.max(delay, 1), MAX_COOLDOWN_MS); } +/** + * True while the given key is inside its 429 cooldown window (lazily evicting the entry once the + * window expires). Used to skip keys that the upstream just rate-limited during failover. + */ function isKeyInCooldown(providerName: string, keyId: string, now = Date.now()): boolean { const entry = keyCooldowns.get(cooldownKey(providerName, keyId)); if (!entry) return false; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 0a9ba4754..51b23691e 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2589,13 +2589,15 @@ async function handleResponsesInner( cancelBodyOnAbort(upstreamResponse.body, upstream.signal); + // Anthropic-only: one bounded internal continuation re-ask for clean end_turn turns that + // announced an edit without emitting a tool call. + const terminalGuardEnabled = activeAdapter.name === "anthropic" && !options.comboAttempt && !routedCompaction; /** * One bounded internal re-ask for Anthropic end_turn-without-tool-call turns. Replays the * continuation on a 429 with the same-key retry budget (hoisted per request), then falls * back to key/account failover; a failure becomes an in-stream adapter error so the client * never sees a second hidden HTTP response or an unbounded retry loop. */ - const terminalGuardEnabled = activeAdapter.name === "anthropic" && !options.comboAttempt && !routedCompaction; const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let imageTierBias = 0; let response: Response | undefined; diff --git a/src/types.ts b/src/types.ts index d478e153b..be12cde96 100644 --- a/src/types.ts +++ b/src/types.ts @@ -924,6 +924,10 @@ export interface RateLimitRetryPolicy { respectRetryAfter?: boolean; } +/** + * One configured provider entry. `authMode` (default `"key"`) decides whether same-target 429 + * retries are allowed; OAuth/forward credentials and local runtimes are never replayed. + */ export interface OcxProviderConfig { adapter: string; /** Cursor MCP compatibility bounds; positive integers when configured. */ diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 42ad9f197..8fbd21127 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -325,6 +325,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { headers: selectedForwardHeaders, diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index 510df4638..b44f97e84 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -62,6 +62,7 @@ describe("usage log", () => { expect(readUsageEntries()[0]?.attempts?.[0]?.recoveryKinds).toEqual(["rate-limit-429"]); }); + /** Build one minimal persisted-usage JSONL line for the given request id. */ const persistedLine = (requestId: string) => JSON.stringify({ requestId, timestamp: 1, From d4e19788217d06026613bb2b76c1c0ba7d8d8dd9 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 04:10:43 +0800 Subject: [PATCH 09/29] fix(proxy): address review round 3 (per-request bridge budget, single send telemetry, invalid master switch, secret-safe warnings) --- src/config.ts | 18 +++++++-- src/images/loop.ts | 22 ++++++----- src/server/responses/core.ts | 8 ++-- src/web-search/loop.ts | 19 ++++++---- tests/config-user-edits.test.ts | 43 ++++++++++++++++++++- tests/images/loop.test.ts | 49 +++++++++++++++++++++++- tests/web-search.test.ts | 67 ++++++++++++++++++++++++++++++++- 7 files changed, 196 insertions(+), 30 deletions(-) diff --git a/src/config.ts b/src/config.ts index 677924541..1b56c5b65 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1129,7 +1129,16 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { if (policy === undefined) continue; if (!policy || typeof policy !== "object" || Array.isArray(policy)) { delete p.retryOn429; - console.warn(`⚠️ config.json providers.${name}.retryOn429 ${JSON.stringify(policy)} is invalid — ignoring the policy`); + // Never serialize the value: an accidental `retryOn429: "sk-..."` would leak the secret. + console.warn(`⚠️ config.json providers.${name}.retryOn429 (${typeof policy}) is invalid — ignoring the policy`); + continue; + } + const policyRecord = policy as Record; + // An explicitly present but invalid master switch must not silently default to ENABLED: + // drop the whole policy so a hand-edit that tried to disable retries stays disabled. + if ("enabled" in policyRecord && typeof policyRecord.enabled !== "boolean") { + delete p.retryOn429; + console.warn(`⚠️ config.json providers.${name}.retryOn429.enabled (${typeof policyRecord.enabled}) is invalid — ignoring the whole policy`); continue; } const fields: Array<[string, (value: unknown) => boolean]> = [ @@ -1141,13 +1150,14 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { ]; const cleaned: Record = {}; for (const [key, isValid] of fields) { - const value = (policy as Record)[key]; + const value = policyRecord[key]; if (value === undefined) continue; if (isValid(value)) cleaned[key] = value; - else console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} ${JSON.stringify(value)} is invalid — ignoring the field`); + // Log only the received type, never the value (provider config can hold secrets). + else console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} (${typeof value}) is invalid — ignoring the field`); } const knownKeys = new Set(fields.map(([key]) => key)); - for (const key of Object.keys(policy as Record)) { + for (const key of Object.keys(policyRecord)) { if (!knownKeys.has(key)) { console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} is not a recognized field — ignoring it`); } diff --git a/src/images/loop.ts b/src/images/loop.ts index 0aceb39d5..9bb979cfa 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -16,6 +16,7 @@ import { pathToFileURL } from "node:url"; import { createAdapterEventQueue } from "../adapters/run-turn-queue"; import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderContinuationState, OcxRequestOptions, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types"; import { namespacedToolName } from "../types"; +import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; @@ -213,8 +214,8 @@ export interface ImageBridgeDeps { videoTimeoutMs?: number; /** Headers forwarded from the original request (e.g. Codex auth). Cloned per iteration. */ forwardHeaders?: Headers; - /** Called before each routed-model dispatch in the bridge loop, for attempt telemetry. */ - onAttemptSend?: () => void; + /** Called before each routed-model dispatch in the bridge loop, for attempt telemetry. Same-target 429 replays pass the `rate-limit-429` recovery kind. */ + onAttemptSend?: (recovery?: AttemptRecoveryKind) => void; /** Called after each upstream request is built (parity with web-search / normal path). */ onRequestBuilt?: (request: AdapterRequest) => void; abortSignal?: AbortSignal; @@ -236,8 +237,6 @@ export interface ImageBridgeDeps { on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; - /** Telemetry for a same-target 429 replay send (records the `rate-limit-429` recovery kind). */ - onRateLimitRetrySend?: () => void; /** Called when the bridged Responses stream completes (parity with runTurn / routed paths). */ onCompletedResponse?: (response: Record, providerState?: OcxProviderContinuationState) => void; /** WebSocket Responses path only — leave response id empty for protocol compatibility. */ @@ -336,6 +335,12 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise; + // Same-target 429 budget is per REQUEST, not per model iteration: later image rounds inherit + // what earlier rounds left of `attempts`, so a bounded multi-round turn can never exceed the + // configured replay count in total (a per-round reset would multiply it by maxRounds). + const rateLimitRetryPolicy = deps.retryOn429Policy ?? null; + let rateLimitRetries = 0; + // Acquire one iteration's final response headers. The first call is drained eagerly so an initial // connect/header/HTTP failure stays a non-2xx JSON response — except for runTurn adapters, which // have no HTTP status surface and must not block SSE headers behind queue.collect(). @@ -438,14 +443,14 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise => { + const fetchOnce = async (requestAdapter: ProviderAdapter, recovery?: AttemptRecoveryKind): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(), abortSignal: headerDeadline.signal, translatorBudget, }); try { deps.onRequestBuilt?.(request); } catch { /* diagnostics are best-effort */ } - deps.onAttemptSend?.(); + deps.onAttemptSend?.(recovery); let response: Response; try { response = requestAdapter.fetchResponse @@ -477,8 +482,6 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens), + onAttemptSend: (recovery?: AttemptRecoveryKind) => + noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, recovery), abortSignal: options.abortSignal, maxRounds: imgPlan && vidPlan ? clampImageMaxRounds(Math.min(config.images?.maxRounds ?? 3, config.images?.videoMaxRounds ?? 2)) @@ -2106,7 +2107,6 @@ async function handleResponsesInner( ); }, retryOn429Policy: rateLimitRetryPolicyFor(route.provider), - onRateLimitRetrySend: () => noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, "rate-limit-429"), ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), ...(options.forceEmptyResponseId ? { forceEmptyResponseId: true } : {}), onCompletedResponse: (response, providerState) => @@ -2135,7 +2135,6 @@ async function handleResponsesInner( // through web-search instead of being swallowed. runTurn adapters never enter this branch. if (canRunWebSearch && wsPlan) { parsed.context.tools = [...(parsed.context.tools ?? []), buildWebSearchTool()]; - noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens); const wsResponse = await runWithWebSearch({ parsed, adapter, incomingMeta: { headers: selectedForwardHeaders, abortSignal: options.abortSignal, translatorBudget }, @@ -2150,6 +2149,8 @@ async function handleResponsesInner( abortSignal: options.abortSignal, ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), onRequestBuilt: request => recordAdapterReasoning(logCtx, request), + onAttemptSend: (recovery?: AttemptRecoveryKind) => + noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, recovery), onUsage: usage => { logCtx.usageFromBridge = true; if (usage) { @@ -2176,7 +2177,6 @@ async function handleResponsesInner( ); }, retryOn429Policy: rateLimitRetryPolicyFor(route.provider), - onRateLimitRetrySend: () => noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, "rate-limit-429"), }); // Register the sidecar stream as an active turn so drainAndShutdown waits for (or aborts) // in-flight web-search turns instead of skipping them during graceful shutdown. diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 8fbd21127..43b079612 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -1,6 +1,7 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../adapters/base"; import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types"; import { namespacedToolName } from "../types"; +import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor"; import { runAnthropicWebSearch } from "./anthropic-executor"; @@ -240,6 +241,8 @@ export interface WebSearchLoopDeps { onUsage?: (usage: OcxUsage | undefined) => void; /** Observe the exact adapter request selected for each routed-model iteration. */ onRequestBuilt?: (request: AdapterRequest) => void; + /** Called before each routed-model dispatch in the loop, for attempt telemetry. Same-target 429 replays pass the `rate-limit-429` recovery kind. */ + onAttemptSend?: (recovery?: AttemptRecoveryKind) => void; /** * 429 key-failover hook: rotate the provider's active pool key and return a rebuilt adapter, * or null when the pool is exhausted (same semantics as the normal routed path). @@ -247,8 +250,6 @@ export interface WebSearchLoopDeps { on429?: (retryAfterHeader: string | null) => ProviderAdapter | null; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; - /** Telemetry for a same-target 429 replay send (records the `rate-limit-429` recovery kind). */ - onRateLimitRetrySend?: () => void; } /** @@ -299,6 +300,12 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise; + // Same-target 429 budget is per REQUEST, not per model iteration: later search rounds inherit + // what earlier rounds left of `attempts`, so a bounded multi-round turn can never exceed the + // configured replay count in total (a per-round reset would multiply it by maxSearches). + const rateLimitRetryPolicy = deps.retryOn429Policy ?? null; + let rateLimitRetries = 0; + // Acquire one iteration's final response headers. The first call is drained eagerly so an initial // connect/header/HTTP failure stays a non-2xx JSON response. Its successful BODY is deliberately // left unread until the downstream Responses SSE bridge exists. @@ -329,7 +336,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise => { + const fetchOnce = async (requestAdapter: ProviderAdapter, recovery?: AttemptRecoveryKind): Promise => { const request = await requestAdapter.buildRequest(iterParsed, { headers: selectedForwardHeaders, abortSignal: headerDeadline.signal, @@ -340,6 +347,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: { enabled: "false", intervalMs: 120 }, + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test).toBeDefined(); + // A hand-edit that tried to disable retries must not become default-ENABLED. + expect(live.providers.test.retryOn429).toBeUndefined(); +}); + +test("invalid retryOn429 values never log the raw value", () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: "sk-super-secret-abc123", + }, + }, + }); + loadConfig(); + const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); + expect(logged).not.toContain("sk-super-secret-abc123"); + expect(logged).toContain("string"); + } finally { + warn.mockRestore(); + } +}); + // R4-1: the request path. A 429 mid-turn rotates a key and saves, with no user action. test("a 429 key rotation does not clobber the hand edit", async () => { const { rotateKeyOn429 } = await import("../src/providers/key-failover"); diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index 0e93d4031..2a58eeee0 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -205,8 +205,8 @@ describe("runWithImageBridge", () => { rotations += 1; return null; }, - onRateLimitRetrySend: () => { - retrySends += 1; + onAttemptSend: recovery => { + if (recovery === "rate-limit-429") retrySends += 1; }, }); const sse = await response.text(); @@ -216,6 +216,51 @@ describe("runWithImageBridge", () => { expect(retrySends).toBe(1); }); + test("retryOn429 budget is shared across iterations (per request, not per round)", async () => { + let sends = 0; + let retrySends = 0; + let rotations = 0; + const retryingAdapter: ProviderAdapter = { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + if (sends === 1 || sends === 3) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + }, + }; + // Round 0: 429 -> one same-key replay (attempts=1) -> 200 carrying an image call. + // Round 1 (forced final): 429 with the request budget already spent -> no replay -> rotation. + streamQueue = [ + [...imageCallEvents], + [{ type: "text_delta" as const, text: "unused" }, { type: "done" as const }], + ]; + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: retryingAdapter, + plan, + maxRounds: 1, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 50, maxIntervalMs: 60_000, respectRetryAfter: false }, + on429: () => { + rotations += 1; + return null; + }, + onAttemptSend: recovery => { + if (recovery === "rate-limit-429") retrySends += 1; + }, + }); + const sse = await response.text(); + expect(sends).toBe(3); + expect(retrySends).toBe(1); + expect(rotations).toBe(1); + // The exhausted final 429 surfaces as the provider error, not a silent success. + expect(sse).toContain("Provider error 429"); + }); + test("forced-final clears named image tool_choice", async () => { streamQueue = [ [{ type: "text_delta" as const, text: "done" }, { type: "done" as const }], diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index 741987b66..52a1263a3 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -668,8 +668,8 @@ describe("web-search sidecar native web_search_call emission", () => { rotations += 1; return null; }, - onRateLimitRetrySend: () => { - retrySends += 1; + onAttemptSend: recovery => { + if (recovery === "rate-limit-429") retrySends += 1; }, }); expect(response.status).toBe(200); @@ -682,6 +682,69 @@ describe("web-search sidecar native web_search_call emission", () => { expect(retrySends).toBe(1); }); + test("retryOn429 budget is shared across iterations (per request, not per round)", async () => { + globalThis.fetch = ((input) => { + const url = String(input); + if (url.startsWith("https://routed.test/")) return Promise.resolve(new Response("{}", { status: 200 })); + // sidecar /responses: return a minimal completed SSE so the search round advances. + return Promise.resolve(new Response( + 'event: response.completed\ndata: {"type":"response.completed"}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + )); + }) as typeof fetch; + + let sends = 0; + let retrySends = 0; + let rotations = 0; + const retryingAdapter: ProviderAdapter = { + name: "mock-retry429", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + sends += 1; + if (sends === 1 || sends === 3) { + return new Response("rate limited", { status: 429, headers: { "retry-after": "30" } }); + } + return new Response("{}", { status: 200 }); + }, + async *parseStream() { + if (sends === 2) { + // Round 0 success carries a web_search call so the loop advances to a forced-answer round. + yield { type: "tool_call_start", id: "call_1", name: "web_search" }; + yield { type: "tool_call_delta", arguments: JSON.stringify({ query: "current docs" }) }; + yield { type: "tool_call_end" }; + } else { + yield { type: "text_delta", text: "unused" }; + } + yield { type: "done" }; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter: retryingAdapter, + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.4-mini", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 50, maxIntervalMs: 60_000, respectRetryAfter: false }, + on429: () => { + rotations += 1; + return null; + }, + onAttemptSend: recovery => { + if (recovery === "rate-limit-429") retrySends += 1; + }, + }); + const frames = await collectSse(response.body!); + expect(sends).toBe(3); + expect(retrySends).toBe(1); + expect(rotations).toBe(1); + const failed = frames.find(f => f.event === "response.failed")?.data.response as { error?: { message?: string } } | undefined; + expect(failed?.error?.message ?? "").toContain("429"); + }); + test("loop 429 with exhausted pool (on429 null) surfaces the provider error", async () => { const firstAdapter: ProviderAdapter = { name: "mock-429", From 58bf694f1e68046a1957673991c42bc7aad19b7b Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 04:33:16 +0800 Subject: [PATCH 10/29] fix(proxy): local audit round (shared request budget, awaited body cancel, post-sleep abort re-checks, pinned xAI req-id) --- src/providers/xai-transport.ts | 7 ++- src/server/responses/core.ts | 44 +++++++++---- tests/images/loop.test.ts | 46 ++++++++++++++ tests/rate-limit-retry.test.ts | 5 ++ tests/terminal-guard-server.test.ts | 98 +++++++++++++++++++++++++++++ tests/xai-transport.test.ts | 8 ++- 6 files changed, 192 insertions(+), 16 deletions(-) diff --git a/src/providers/xai-transport.ts b/src/providers/xai-transport.ts index 2c6060592..5709779fc 100644 --- a/src/providers/xai-transport.ts +++ b/src/providers/xai-transport.ts @@ -128,9 +128,14 @@ export function resolveProviderTransport( provider.headers, XAI_GROK_COMPATIBILITY.headers.requestId, ); + // Pin the request id per resolved transport (= per logical request until key rotation): + // same-target 429 replays must carry the SAME x-grok-req-id as the original dispatch, and + // transient retries reuse one id so the upstream can dedupe them. A rotated key resolves a + // fresh transport, which gets its own id. + const requestId = configuredRequestId ?? randomUUID(); const baseFetch = provider.fetch ?? globalThis.fetch; const attemptFetch = ((input, init) => - baseFetch(input, withGeneratedRequestId(init, configuredRequestId, stableHeaders))) as typeof globalThis.fetch; + baseFetch(input, withGeneratedRequestId(init, requestId, stableHeaders))) as typeof globalThis.fetch; return { ...provider, diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index ab26b1f40..5fb47d95b 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -87,7 +87,6 @@ import { } from "../../codex/routing"; import { applyUpstreamRecoveryInit, - cancelResponseBodyBestEffort, fetchWithResetRetry, fetchWithTransientRetry, sleepWithAbort, @@ -1656,7 +1655,8 @@ async function handleResponsesInner( rateLimitRetries += 1; // Release the unread 429 body before the backoff (only the header is needed for the wait). const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - cancelResponseBodyBestEffort(upstreamResponse); + // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. + try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -1666,6 +1666,12 @@ async function handleResponsesInner( upstream.abort(); return clientCancelledResponse(); } + // Client cancellation wins over any stale timer edge: re-check before dispatching the + // replay so the wire never starts work for a request the client already abandoned. + if (options.abortSignal?.aborted || upstream.signal.aborted) { + upstream.abort(); + return clientCancelledResponse(); + } try { upstreamResponse = await fetchWithTransientRetry( recovery => { @@ -2355,6 +2361,12 @@ async function handleResponsesInner( request.releaseBodyObservation?.(); } + // Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401 + // replay that comes back 429 cannot silently re-arm a fresh budget) and is SHARED with the + // terminal-guard continuation below, so the main loop + one continuation can never exceed + // `attempts` same-key replays in total (bounded per request). + const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); + let rateLimitRetries = 0; if (!upstreamResponse.ok) { // Recovery loop: multi-key 429 failover + at most ONE anthropic 413 tightened retry // (devlog/260714_image_normalization_pipeline/030). One mutable activeAdapter serves @@ -2364,10 +2376,6 @@ async function handleResponsesInner( let imageTierBias = 0; let imageRetryAttempted = false; let oauth401ReplayAttempted = false; - // Same-target 429 retry budget lives OUTSIDE the recovery loop so a 413/401 replay that - // comes back 429 cannot silently re-arm a fresh budget (bounded to `attempts` per request). - const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); - let rateLimitRetries = 0; /** * Rebuild the request from the current parsed input (and any image-tier bias) and refetch * it once, tagging the attempt with the given recovery kind. Rebuilds are deterministic @@ -2462,7 +2470,8 @@ async function handleResponsesInner( // wait, and under a 429 storm the sockets would otherwise accumulate for the whole // configured interval (same pattern as the key-failover branch below). const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - cancelResponseBodyBestEffort(upstreamResponse); + // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. + try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -2473,6 +2482,13 @@ async function handleResponsesInner( upstream.abort(); return clientCancelledResponse(); } + // Client cancellation wins over any stale timer edge: re-check before dispatching the + // replay so an adapter never starts work for a request the client already abandoned. + if (options.abortSignal?.aborted || upstream.signal.aborted) { + cleanupUpstreamAbort(); + upstream.abort(); + return clientCancelledResponse(); + } const result = await rebuildAndRefetch("rate-limit-429"); if ("failed" in result) return result.failed; upstreamResponse = result; @@ -2601,11 +2617,6 @@ async function handleResponsesInner( const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let imageTierBias = 0; let response: Response | undefined; - // Same-target 429 budget is per REQUEST, not per failover iteration: a key/account rotation - // or a 413 tier retry that comes back 429 must not re-arm a fresh budget (parity with the - // main recovery loop above). - const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); - let rateLimitRetries = 0; /** * Build and fetch one terminal-guard continuation. `replay` marks a same-target 429 * replay so its send records the `rate-limit-429` recovery kind; the adapter rebuild is @@ -2676,7 +2687,8 @@ async function handleResponsesInner( rateLimitRetries += 1; // Release the unread 429 body before the backoff (only the header is needed for the wait). const retryAfterHeader = response.headers.get("retry-after"); - try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. + try { await response.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -2693,6 +2705,12 @@ async function handleResponsesInner( } return; } + // Client cancellation wins over any stale timer edge: re-check before dispatching the + // replay so the continuation never starts work for a request the client abandoned. + if (options.abortSignal?.aborted || upstream.signal.aborted) { + yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; + return; + } try { response = await fetchContinuation(true); } catch (error) { diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index 2a58eeee0..e180babce 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -261,6 +261,52 @@ describe("runWithImageBridge", () => { expect(sse).toContain("Provider error 429"); }); + test("retryOn429 budget is not re-armed after on429 rotation returns a new adapter", async () => { + let sends = 0; + let retrySends = 0; + let rotations = 0; + const retryingAdapter: ProviderAdapter = { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + }, + }; + streamQueue = [[{ type: "text_delta" as const, text: "unused" }, { type: "done" as const }]]; + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: retryingAdapter, + plan, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 50, maxIntervalMs: 60_000, respectRetryAfter: false }, + on429: () => { + rotations += 1; + // First rotation returns a new adapter that also 429s; the exhausted budget must not + // re-arm for it. Second call returns null to terminate the pool. + return rotations === 1 + ? ({ + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + return new Response("{}", { status: 429 }); + }, + } as ProviderAdapter) + : null; + }, + onAttemptSend: recovery => { + if (recovery === "rate-limit-429") retrySends += 1; + }, + }); + const sse = await response.text(); + // initial 429 + 1 same-key replay + 1 rotated send (no replay on the rotated adapter) = 3. + expect(sends).toBe(3); + expect(retrySends).toBe(1); + expect(rotations).toBe(2); + expect(sse).toContain("Provider error 429"); + }); + test("forced-final clears named image tool_choice", async () => { streamQueue = [ [{ type: "text_delta" as const, text: "done" }, { type: "done" as const }], diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts index 256bdb65e..6db97769c 100644 --- a/tests/rate-limit-retry.test.ts +++ b/tests/rate-limit-retry.test.ts @@ -82,6 +82,11 @@ describe("rateLimitRetryDelayMs", () => { expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:28:00 GMT", now)).toBe(1); }); + test("a far-future HTTP-date Retry-After is capped at maxIntervalMs", () => { + const now = Date.parse("2026-10-21T07:27:30Z"); + expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2027 07:28:00 GMT", now)).toBe(60_000); + }); + test("Retry-After 0 retries immediately instead of falling back to the interval", () => { expect(rateLimitRetryDelayMs(policy, "0", 1_000_000)).toBe(1); }); diff --git a/tests/terminal-guard-server.test.ts b/tests/terminal-guard-server.test.ts index e52eb1d9b..721081611 100644 --- a/tests/terminal-guard-server.test.ts +++ b/tests/terminal-guard-server.test.ts @@ -172,4 +172,102 @@ describe("server terminal guard integration", () => { expect(sends).toBe(4); }); + test("terminal-guard continuation shares the request-wide 429 budget with the main loop", async () => { + const budgetConfig = { + ...config, + providers: { + "claude-se": { + adapter: "anthropic", + baseUrl: "https://example.test", + apiKey: "sk-test", + retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false }, + }, + }, + } as unknown as OcxConfig; + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) { + // The main recovery loop consumes the only same-key replay... + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + if (sends === 2) { + // ...the replay succeeds and the terminal-guard continuation starts. + return anthropicSse(firstTurn); + } + // Continuation 429 with the request budget already spent: surfaces, never replays. + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "se-claude-opus-4.8", + input: "请检查这个问题并修复代码", + stream: true, + tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }], + }), + }), budgetConfig, { model: "", provider: "" }); + + const text = await response.text(); + // main 429 -> same-key replay -> continuation 429 (no budget left) = exactly 3 sends. + expect(sends).toBe(3); + expect(text).toContain("Provider continuation error 429"); + }); + + test("terminal-guard continuation abort during the 429 wait yields 499 without replaying", async () => { + const abortConfig = { + ...config, + providers: { + "claude-se": { + adapter: "anthropic", + baseUrl: "https://example.test", + apiKey: "sk-test", + retryOn429: { attempts: 3, intervalMs: 30_000, respectRetryAfter: false }, + }, + }, + } as unknown as OcxConfig; + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) { + return anthropicSse(firstTurn); + } + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + + const abort = new AbortController(); + const pending = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "se-claude-opus-4.8", + input: "请检查这个问题并修复代码", + stream: true, + tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }], + }), + }), abortConfig, { model: "", provider: "" }, { abortSignal: abort.signal }); + + // The terminal-guard continuation runs inside the SSE producer, so consume the body to + // drive it, then wait until the continuation's 429 lands and its retry sleep is running. + const response = await pending; + const textPromise = response.text(); + for (let i = 0; i < 100 && sends < 2; i += 1) await Bun.sleep(10); + expect(sends).toBe(2); + abort.abort(new DOMException("client disconnected", "AbortError")); + const text = await textPromise; + expect(text).toContain("client closed request during terminal continuation"); + expect(sends).toBe(2); + }); + }); diff --git a/tests/xai-transport.test.ts b/tests/xai-transport.test.ts index ed71080f8..e999f1763 100644 --- a/tests/xai-transport.test.ts +++ b/tests/xai-transport.test.ts @@ -294,12 +294,16 @@ describe("xAI outbound compatibility headers", () => { ]) expect(seen[0].has(name)).toBe(false); }); - test("same resolved transport refreshes req-id but keeps conv-id stable", async () => { + test("same resolved transport pins one req-id per logical request (identical replays) and keeps conv-id stable", async () => { const { seen } = await capture("oauth", 2); expect(seen).toHaveLength(2); expect(seen[0].get("x-grok-req-id")).toMatch(UUID_V4); expect(seen[1].get("x-grok-req-id")).toMatch(UUID_V4); - expect(seen[1].get("x-grok-req-id")).not.toBe(seen[0].get("x-grok-req-id")); + // A same-target 429 replay must be byte-identical, including x-grok-req-id; a new resolve + // (e.g. after key rotation) produces a fresh transport and therefore a fresh id. + expect(seen[1].get("x-grok-req-id")).toBe(seen[0].get("x-grok-req-id")); + const freshTransport = await capture("oauth", 1); + expect(freshTransport.seen[0].get("x-grok-req-id")).not.toBe(seen[0].get("x-grok-req-id")); expect(seen[0].get("x-grok-conv-id")).toBe(deriveXaiConvId("codex-session-abc")); expect(seen[1].get("x-grok-conv-id")).toBe(seen[0].get("x-grok-conv-id")); expect(seen[1].get("x-grok-session-id")).toBe(seen[0].get("x-grok-session-id")); From 4fee87b530b8b29d41f93c2f6b4c7dde6a82980d Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 04:39:19 +0800 Subject: [PATCH 11/29] fix(config): redact unrecognized retryOn429 field names in load warnings --- src/config.ts | 6 +++++- tests/config-user-edits.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 1b56c5b65..b97bec7e6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,6 +23,7 @@ import { import { recordOwnedConfigPath } from "./lib/config-ownership"; import { assertNotRealHomeUnderTest } from "./lib/test-home-guard"; import { providerDestinationConfigError } from "./lib/destination-policy"; +import { redactSecretString } from "./lib/redact"; import { openRouterRoutingConfigError } from "./providers/openrouter-routing"; import { isWirePinnedModel, @@ -1159,7 +1160,10 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { const knownKeys = new Set(fields.map(([key]) => key)); for (const key of Object.keys(policyRecord)) { if (!knownKeys.has(key)) { - console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} is not a recognized field — ignoring it`); + // Redact the field NAME before logging: a malformed hand-edit can place a secret in a + // property name (`retryOn429: { "sk-...": true }`). Ordinary typos (e.g. `attempt`) + // stay readable, secret-shaped names become [REDACTED]. + console.warn(`⚠️ config.json providers.${name}.retryOn429.${redactSecretString(key)} is not a recognized field — ignoring it`); } } p.retryOn429 = cleaned; diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 4d2baf716..ed70455d0 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -188,6 +188,31 @@ test("invalid retryOn429 values never log the raw value", () => { } }); +test("unrecognized retryOn429 field names are redacted before logging", () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: { "sk-super-secret-keyname-987654": true, intervalMs: 120 }, + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test.retryOn429).toEqual({ intervalMs: 120 }); + const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); + // The secret-shaped property NAME must never reach the log; the valid field survives. + expect(logged).not.toContain("sk-super-secret-keyname-987654"); + expect(logged).toContain("[REDACTED]"); + } finally { + warn.mockRestore(); + } +}); + // R4-1: the request path. A 429 mid-turn rotates a key and saves, with no user action. test("a 429 key rotation does not clobber the hand edit", async () => { const { rotateKeyOn429 } = await import("../src/providers/key-failover"); From 5945711bfc6fca1a781f637082e542476b599ef8 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 04:41:18 +0800 Subject: [PATCH 12/29] refactor(xai): rename pinned request-id param; tighten secret-warning test assertion --- src/providers/xai-transport.ts | 4 ++-- tests/config-user-edits.test.ts | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/providers/xai-transport.ts b/src/providers/xai-transport.ts index 5709779fc..b08d12798 100644 --- a/src/providers/xai-transport.ts +++ b/src/providers/xai-transport.ts @@ -53,7 +53,7 @@ function withoutUserOverridden( function withGeneratedRequestId( init: RequestInit | undefined, - configuredRequestId: string | undefined, + pinnedRequestId: string, stableHeaders: Readonly>, ): RequestInit { const headers = new Headers(init?.headers); @@ -63,7 +63,7 @@ function withGeneratedRequestId( if (!headers.has(XAI_GROK_COMPATIBILITY.headers.requestId)) { headers.set( XAI_GROK_COMPATIBILITY.headers.requestId, - configuredRequestId ?? randomUUID(), + pinnedRequestId, ); } return { ...init, headers }; diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index ed70455d0..4c891f84b 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -182,7 +182,8 @@ test("invalid retryOn429 values never log the raw value", () => { loadConfig(); const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); expect(logged).not.toContain("sk-super-secret-abc123"); - expect(logged).toContain("string"); + // Anchor the type-only diagnostic to the exact field so unrelated warnings can't satisfy it. + expect(logged).toContain("providers.test.retryOn429 (string) is invalid"); } finally { warn.mockRestore(); } @@ -198,7 +199,7 @@ test("unrecognized retryOn429 field names are redacted before logging", () => { baseUrl: "http://127.0.0.1:1/v1", apiKey: "k", allowPrivateNetwork: true, - retryOn429: { "sk-super-secret-keyname-987654": true, intervalMs: 120 }, + retryOn429: { "sk-super-secret-9876": true, intervalMs: 120 }, }, }, }); @@ -206,7 +207,7 @@ test("unrecognized retryOn429 field names are redacted before logging", () => { expect(live.providers.test.retryOn429).toEqual({ intervalMs: 120 }); const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); // The secret-shaped property NAME must never reach the log; the valid field survives. - expect(logged).not.toContain("sk-super-secret-keyname-987654"); + expect(logged).not.toContain("sk-super-secret-9876"); expect(logged).toContain("[REDACTED]"); } finally { warn.mockRestore(); From e65106f0484300e99be5e565e2787a4f2518062b Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 04:48:13 +0800 Subject: [PATCH 13/29] fix(config): JSON-escape redacted retryOn429 field names in load warnings --- src/config.ts | 5 +++-- tests/config-user-edits.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index b97bec7e6..ac3448fb8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1162,8 +1162,9 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { if (!knownKeys.has(key)) { // Redact the field NAME before logging: a malformed hand-edit can place a secret in a // property name (`retryOn429: { "sk-...": true }`). Ordinary typos (e.g. `attempt`) - // stay readable, secret-shaped names become [REDACTED]. - console.warn(`⚠️ config.json providers.${name}.retryOn429.${redactSecretString(key)} is not a recognized field — ignoring it`); + // stay readable, secret-shaped names become [REDACTED]. JSON-escape afterwards so a + // control-character property name (newline/ANSI) can never forge a log line. + console.warn(`⚠️ config.json providers.${name}.retryOn429.${JSON.stringify(redactSecretString(key))} is not a recognized field — ignoring it`); } } p.retryOn429 = cleaned; diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 4c891f84b..b933a1be6 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -214,6 +214,32 @@ test("unrecognized retryOn429 field names are redacted before logging", () => { } }); +test("unrecognized retryOn429 field names are JSON-escaped before logging", () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: { "evil\nattempt": true, intervalMs: 120 }, + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test.retryOn429).toEqual({ intervalMs: 120 }); + const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); + // The raw control character must never reach the log (no line forging); the escaped form + // still names the field for typo debugging. + expect(logged).not.toContain("evil\nattempt"); + expect(logged).toContain('"evil\\nattempt"'); + } finally { + warn.mockRestore(); + } +}); + // R4-1: the request path. A 429 mid-turn rotates a key and saves, with no user action. test("a 429 key rotation does not clobber the hand edit", async () => { const { rotateKeyOn429 } = await import("../src/providers/key-failover"); From 89535fb78d4d96c9a5d26758b1f6ade95c9b3ef1 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 05:16:41 +0800 Subject: [PATCH 14/29] fix(config): redact and JSON-escape provider names in retryOn429 load warnings --- src/config.ts | 11 ++++--- tests/config-user-edits.test.ts | 51 ++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index ac3448fb8..aa6e5a428 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1124,6 +1124,9 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { const providers = root.providers; if (!providers || typeof providers !== "object" || Array.isArray(providers)) return; for (const [name, provider] of Object.entries(providers as Record)) { + // This sanitizer runs BEFORE schema validation, so the provider name is untrusted: redact + // secret-shaped names and JSON-escape control characters before it reaches any warning. + const safeProviderName = JSON.stringify(redactSecretString(name)); if (!provider || typeof provider !== "object" || Array.isArray(provider)) continue; const p = provider as Record; const policy = p.retryOn429; @@ -1131,7 +1134,7 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { if (!policy || typeof policy !== "object" || Array.isArray(policy)) { delete p.retryOn429; // Never serialize the value: an accidental `retryOn429: "sk-..."` would leak the secret. - console.warn(`⚠️ config.json providers.${name}.retryOn429 (${typeof policy}) is invalid — ignoring the policy`); + console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429 (${typeof policy}) is invalid — ignoring the policy`); continue; } const policyRecord = policy as Record; @@ -1139,7 +1142,7 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { // drop the whole policy so a hand-edit that tried to disable retries stays disabled. if ("enabled" in policyRecord && typeof policyRecord.enabled !== "boolean") { delete p.retryOn429; - console.warn(`⚠️ config.json providers.${name}.retryOn429.enabled (${typeof policyRecord.enabled}) is invalid — ignoring the whole policy`); + console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.enabled (${typeof policyRecord.enabled}) is invalid — ignoring the whole policy`); continue; } const fields: Array<[string, (value: unknown) => boolean]> = [ @@ -1155,7 +1158,7 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { if (value === undefined) continue; if (isValid(value)) cleaned[key] = value; // Log only the received type, never the value (provider config can hold secrets). - else console.warn(`⚠️ config.json providers.${name}.retryOn429.${key} (${typeof value}) is invalid — ignoring the field`); + else console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.${key} (${typeof value}) is invalid — ignoring the field`); } const knownKeys = new Set(fields.map(([key]) => key)); for (const key of Object.keys(policyRecord)) { @@ -1164,7 +1167,7 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { // property name (`retryOn429: { "sk-...": true }`). Ordinary typos (e.g. `attempt`) // stay readable, secret-shaped names become [REDACTED]. JSON-escape afterwards so a // control-character property name (newline/ANSI) can never forge a log line. - console.warn(`⚠️ config.json providers.${name}.retryOn429.${JSON.stringify(redactSecretString(key))} is not a recognized field — ignoring it`); + console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.${JSON.stringify(redactSecretString(key))} is not a recognized field — ignoring it`); } } p.retryOn429 = cleaned; diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index b933a1be6..2c66c97d3 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -183,7 +183,7 @@ test("invalid retryOn429 values never log the raw value", () => { const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); expect(logged).not.toContain("sk-super-secret-abc123"); // Anchor the type-only diagnostic to the exact field so unrelated warnings can't satisfy it. - expect(logged).toContain("providers.test.retryOn429 (string) is invalid"); + expect(logged).toContain('providers."test".retryOn429 (string) is invalid'); } finally { warn.mockRestore(); } @@ -240,6 +240,55 @@ test("unrecognized retryOn429 field names are JSON-escaped before logging", () = } }); +test("provider names are redacted before retryOn429 load warnings", () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + writeDiskConfig({ + providers: { + "sk-super-secret-9876": { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: "enabled", + }, + }, + }); + loadConfig(); + const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); + // The sanitizer runs before schema validation, so a secret-shaped provider NAME must + // never reach the log either. + expect(logged).not.toContain("sk-super-secret-9876"); + expect(logged).toContain("[REDACTED]"); + } finally { + warn.mockRestore(); + } +}); + +test("provider names with control characters are JSON-escaped before retryOn429 load warnings", () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + writeDiskConfig({ + providers: { + "evil\nprovider": { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: "enabled", + }, + }, + }); + loadConfig(); + const logged = warn.mock.calls.map(call => call.join(" ")).join("\n"); + // The raw newline must never forge a log line; the escaped form still names the provider. + expect(logged).not.toContain("evil\nprovider"); + expect(logged).toContain('"evil\\nprovider"'); + } finally { + warn.mockRestore(); + } +}); + // R4-1: the request path. A 429 mid-turn rotates a key and saves, with no user action. test("a 429 key rotation does not clobber the hand edit", async () => { const { rotateKeyOn429 } = await import("../src/providers/key-failover"); From eb9890e51533e9624b24d872cf64ff774aef131f Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 14:41:25 +0800 Subject: [PATCH 15/29] fix(proxy): maintainer review round (watchdog-fed backoffs, one cached request per target, deadline/stall regression tests) --- src/images/loop.ts | 28 ++++-- src/lib/upstream-retry.ts | 21 +++++ src/server/responses/core.ts | 90 ++++++++++++++----- src/web-search/loop.ts | 40 ++++++--- tests/images/loop.test.ts | 64 ++++++++++++++ tests/server-rate-limit-retry-e2e.test.ts | 10 +++ tests/terminal-guard-server.test.ts | 47 ++++++++++ tests/web-search.test.ts | 103 ++++++++++++++++++++-- 8 files changed, 354 insertions(+), 49 deletions(-) diff --git a/src/images/loop.ts b/src/images/loop.ts index 9bb979cfa..55df17b2f 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -20,7 +20,7 @@ import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry"; +import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -442,14 +442,25 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise => { - const request = await requestAdapter.buildRequest(iterParsed, { - headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(), - abortSignal: headerDeadline.signal, - translatorBudget, - }); - try { deps.onRequestBuilt?.(request); } catch { /* diagnostics are best-effort */ } + let request: AdapterRequest; + if (cachedRequest !== undefined && cachedAdapter === requestAdapter) { + request = cachedRequest; + } else { + request = await requestAdapter.buildRequest(iterParsed, { + headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(), + abortSignal: headerDeadline.signal, + translatorBudget, + }); + try { deps.onRequestBuilt?.(request); } catch { /* diagnostics are best-effort */ } + cachedRequest = request; + cachedAdapter = requestAdapter; + } deps.onAttemptSend?.(recovery); let response: Response; try { @@ -496,9 +507,10 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise { + let remaining = ms; + while (remaining > 0) { + const chunk = Math.min(remaining, heartbeatIntervalMs); + await sleepWithAbort(chunk, signal); + remaining -= chunk; + yield { type: "heartbeat" }; + } +} + export function isConnectionResetError(err: unknown): boolean { if (!(err instanceof Error)) return false; // Aborts and timeouts are caller decisions / honest failures — never retryable. diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 5fb47d95b..9415d40de 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -89,6 +89,7 @@ import { applyUpstreamRecoveryInit, fetchWithResetRetry, fetchWithTransientRetry, + sleepWithHeartbeats, sleepWithAbort, } from "../../lib/upstream-retry"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; @@ -101,6 +102,7 @@ import { isUsageDebugEnabled } from "../../usage/debug"; import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress"; import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve"; import type { InboundWire } from "../../providers/registry"; +import type { AdapterRequest } from "../../adapters/base"; import { hasKeyPoolFailover, rateLimitRetryDelayMs, @@ -2321,19 +2323,33 @@ async function handleResponsesInner( const upstream = new AbortController(); const cleanupUpstreamAbort = linkAbortSignal(upstream, options.abortSignal); const connectMs = config.connectTimeoutMs ?? 200_000; + // Bridge stall budget (seconds of silence before upstream_stall_timeout); the retry backoff + // heartbeat interval is derived from it so the watchdog is always fed during deliberate waits. + const stallTimeoutMs = typeof config.stallTimeoutSec === "number" && Number.isFinite(config.stallTimeoutSec) && config.stallTimeoutSec > 0 + ? Math.floor(config.stallTimeoutSec * 1000) + : 300_000; let activeAdapter = adapter; - const request = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); - recordAdapterReasoning(logCtx, request); - const inputTokenEstimate = typeof request.usageLog?.inputTokens === "number" - ? request.usageLog.inputTokens + // One immutable, body-safe outbound request per same-target sequence (URL, serialized body, + // auth headers, generated compat headers). Same-target 429 replays reuse it verbatim; the + // builder runs again only after a key/account/adapter rotation, an oauth refresh, or an + // image-tier bias change (transportToken bump). `body` is always a serialized string, so + // reuse is safe, and releaseBodyObservation is idempotent per build. + const initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + recordAdapterReasoning(logCtx, initialRequest); + const inputTokenEstimate = typeof initialRequest.usageLog?.inputTokens === "number" + ? initialRequest.usageLog.inputTokens : undefined; if (inputTokenEstimate !== undefined) logCtx.usageLogInputTokens = inputTokenEstimate; + let sameTargetRequest: AdapterRequest | undefined = initialRequest; + let sameTargetParsed: OcxParsedRequest | undefined = parsed; + let sameTargetToken = 0; + let transportToken = 0; let upstreamResponse: Response; try { if (activeAdapter.fetchResponse) { noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate); - upstreamResponse = await activeAdapter.fetchResponse(request, { + upstreamResponse = await activeAdapter.fetchResponse(initialRequest, { abortSignal: upstream.signal, timeoutMs: connectMs, stream: parsed.stream, @@ -2342,13 +2358,13 @@ async function handleResponsesInner( upstreamResponse = await fetchWithResetRetry( recovery => { noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate, recovery); - return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({ - method: request.method, - headers: request.headers, - body: request.body, + return fetchWithHeaderTimeout(initialRequest.url, applyUpstreamRecoveryInit({ + method: initialRequest.method, + headers: initialRequest.headers, + body: initialRequest.body, }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)); }, - { abortSignal: upstream.signal, label: safeHostLabel(request.url) }, + { abortSignal: upstream.signal, label: safeHostLabel(initialRequest.url) }, ); } } catch (err) { @@ -2358,7 +2374,7 @@ async function handleResponsesInner( const msg = describeUpstreamConnectFailure(err, connectMs); return formatErrorResponse(502, "upstream_error", msg); } finally { - request.releaseBodyObservation?.(); + initialRequest.releaseBodyObservation?.(); } // Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401 @@ -2384,12 +2400,21 @@ async function handleResponsesInner( const rebuildAndRefetch = async ( recovery: AttemptRecoveryKind, ): Promise => { - const retryRequest = await activeAdapter.buildRequest(parsed, { - headers: selectedForwardHeaders, - translatorBudget, - ...(imageTierBias > 0 ? { imageTierBias } : {}), - }); - recordAdapterReasoning(logCtx, retryRequest); + let retryRequest: AdapterRequest; + if (sameTargetRequest !== undefined && sameTargetParsed === parsed && sameTargetToken === transportToken) { + // Same target (key/adapter/parsed/tier unchanged): replay the exact cached request. + retryRequest = sameTargetRequest; + } else { + retryRequest = await activeAdapter.buildRequest(parsed, { + headers: selectedForwardHeaders, + translatorBudget, + ...(imageTierBias > 0 ? { imageTierBias } : {}), + }); + recordAdapterReasoning(logCtx, retryRequest); + sameTargetRequest = retryRequest; + sameTargetParsed = parsed; + sameTargetToken = transportToken; + } const retryEstimate = typeof retryRequest.usageLog?.inputTokens === "number" ? retryRequest.usageLog.inputTokens : undefined; @@ -2444,6 +2469,7 @@ async function handleResponsesInner( route.providerName === "github-copilot" ? getOAuthCredentialApiBaseUrl(route.providerName) : undefined, ); route.provider = refreshedProvider; + transportToken += 1; activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire), config.cacheRetention, @@ -2509,6 +2535,7 @@ async function handleResponsesInner( // until runtime cleanup (one per rotated key under a rate-limit storm). try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } route.provider = rotated; + transportToken += 1; activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire), config.cacheRetention, @@ -2539,6 +2566,7 @@ async function handleResponsesInner( anthropicPoolAccountId = nextAccountId; anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: accessToken }; + transportToken += 1; promoteAnthropicActiveAccount(nextAccountId); logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config); activeAdapter = resolveAdapter( @@ -2563,6 +2591,7 @@ async function handleResponsesInner( })) { imageRetryAttempted = true; imageTierBias = 1; + transportToken += 1; try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } const result = await rebuildAndRefetch("image-413"); if ("failed" in result) return result.failed; @@ -2623,12 +2652,21 @@ async function handleResponsesInner( * deterministic for the same parsed request (tests assert byte-identical replays). */ const fetchContinuation = async (replay = false): Promise => { - const continuationRequest = await activeAdapter.buildRequest(nextParsed, { - headers: selectedForwardHeaders, - translatorBudget, - ...(imageTierBias > 0 ? { imageTierBias } : {}), - }); - recordAdapterReasoning(logCtx, continuationRequest); + let continuationRequest: AdapterRequest; + if (sameTargetRequest !== undefined && sameTargetParsed === nextParsed && sameTargetToken === transportToken) { + // Same target (key/adapter/parsed/tier unchanged): replay the exact cached request. + continuationRequest = sameTargetRequest; + } else { + continuationRequest = await activeAdapter.buildRequest(nextParsed, { + headers: selectedForwardHeaders, + translatorBudget, + ...(imageTierBias > 0 ? { imageTierBias } : {}), + }); + recordAdapterReasoning(logCtx, continuationRequest); + sameTargetRequest = continuationRequest; + sameTargetParsed = nextParsed; + sameTargetToken = transportToken; + } const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number" ? continuationRequest.usageLog.inputTokens : undefined; @@ -2690,12 +2728,13 @@ async function handleResponsesInner( // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. try { await response.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { - await sleepWithAbort( + yield* sleepWithHeartbeats( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), // Listen on the upstream signal: once the SSE body is being streamed, a client // cancel aborts `upstream` through the bridge, and upstream is also linked from // options.abortSignal — so this covers both cancellation paths. upstream.signal, + Math.min(10_000, Math.max(250, stallTimeoutMs / 2)), ); } catch { if (options.abortSignal?.aborted || upstream.signal.aborted) { @@ -2733,6 +2772,7 @@ async function handleResponsesInner( if (rotated) { try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } route.provider = rotated; + transportToken += 1; activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire), config.cacheRetention, @@ -2759,6 +2799,7 @@ async function handleResponsesInner( anthropicPoolAccountId = nextAccountId; anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: accessToken }; + transportToken += 1; promoteAnthropicActiveAccount(nextAccountId); logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config); activeAdapter = resolveAdapter( @@ -2779,6 +2820,7 @@ async function handleResponsesInner( alreadyAttempted: imageTierBias > 0, })) { imageTierBias = 1; + transportToken += 1; try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } continue; } diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 43b079612..a6fa4873f 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -8,7 +8,7 @@ import { runAnthropicWebSearch } from "./anthropic-executor"; import { clearableDeadline } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry"; +import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -266,6 +266,12 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise 0 + ? Math.floor(deps.stallTimeoutSec * 1000) + : 300_000; + const messages: OcxMessage[] = [...parsed.context.messages]; const loopT0 = Date.now(); const allTools = parsed.context.tools ?? []; @@ -335,17 +341,28 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise => { - const request = await requestAdapter.buildRequest(iterParsed, { - headers: selectedForwardHeaders, - abortSignal: headerDeadline.signal, - translatorBudget, - }); - try { - deps.onRequestBuilt?.(request); - } catch { - // Diagnostics are best-effort and must never abort a web-search iteration. + let request: AdapterRequest; + if (cachedRequest !== undefined && cachedAdapter === requestAdapter) { + request = cachedRequest; + } else { + request = await requestAdapter.buildRequest(iterParsed, { + headers: selectedForwardHeaders, + abortSignal: headerDeadline.signal, + translatorBudget, + }); + try { + deps.onRequestBuilt?.(request); + } catch { + // Diagnostics are best-effort and must never abort a web-search iteration. + } + cachedRequest = request; + cachedAdapter = requestAdapter; } deps.onAttemptSend?.(recovery); let response: Response; @@ -393,9 +410,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { expect(sends).toBe(2); expect(rotations).toBe(0); expect(retrySends).toBe(1); + // Same-target replay reuses the ONE built request (builder runs once per target sequence). + expect(buildRequestCalls).toBe(1); }); + test("retry wait longer than the stall budget still succeeds (heartbeats feed the watchdog)", async () => { + let sends = 0; + const retryingAdapter: ProviderAdapter = { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + }, + }; + streamQueue = [[{ type: "text_delta" as const, text: "recovered" }, { type: "done" as const }]]; + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: retryingAdapter, + plan, + stallTimeoutSec: 1, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 1_500, maxIntervalMs: 60_000, respectRetryAfter: false }, + }); + const sse = await response.text(); + // A 1.5s backoff under a 1s stall budget must not trip upstream_stall_timeout: the wait + // yields heartbeat events, the replay lands, and the turn completes. + expect(sends).toBe(2); + expect(sse).toContain("recovered"); + expect(sse).not.toContain("upstream_stall_timeout"); + }, 5_000); + + test("retry wait longer than connectTimeoutMs restarts the header deadline (no 504)", async () => { + let sends = 0; + const retryingAdapter: ProviderAdapter = { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + }, + }; + streamQueue = [[{ type: "text_delta" as const, text: "recovered" }, { type: "done" as const }]]; + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: retryingAdapter, + plan, + connectTimeoutMs: 100, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 150, maxIntervalMs: 60_000, respectRetryAfter: false }, + }); + const sse = await response.text(); + // The deliberate backoff must not consume the response-header deadline: a fresh deadline is + // armed after the wait, so the replay gets a new connect budget instead of a 504. + expect(sends).toBe(2); + expect(sse).toContain("recovered"); + expect(sse).not.toContain("504"); + }, 5_000); + test("retryOn429 budget is shared across iterations (per request, not per round)", async () => { let sends = 0; let retrySends = 0; diff --git a/tests/server-rate-limit-retry-e2e.test.ts b/tests/server-rate-limit-retry-e2e.test.ts index f64b80dfe..6a6d7f4c6 100644 --- a/tests/server-rate-limit-retry-e2e.test.ts +++ b/tests/server-rate-limit-retry-e2e.test.ts @@ -49,10 +49,16 @@ describe("server same-target 429 retry (end-to-end)", () => { test("single-key provider replays the identical request until upstream succeeds", async () => { const originalFetch = globalThis.fetch; const seenBodies: string[] = []; + const seenHeaders: string[][] = []; globalThis.fetch = (async (input, init) => { const url = input instanceof Request ? input.url : String(input); if (url === "https://llmapi.blsc.cn/chat/completions") { seenBodies.push(String(init?.body)); + seenHeaders.push( + [...new Headers(init?.headers).entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .flat(), + ); if (seenBodies.length <= 2) { return new Response(JSON.stringify({ error: { message: "rate limited" } }), { status: 429, @@ -89,6 +95,10 @@ describe("server same-target 429 retry (end-to-end)", () => { expect(seenBodies).toHaveLength(3); expect(seenBodies[0]).toBe(seenBodies[1]); expect(seenBodies[1]).toBe(seenBodies[2]); + // Same-target replays reuse the ONE built request: full header set identical too. + expect(seenHeaders).toHaveLength(3); + expect(seenHeaders[0]).toEqual(seenHeaders[1]); + expect(seenHeaders[1]).toEqual(seenHeaders[2]); } finally { server?.stop(true); globalThis.fetch = originalFetch; diff --git a/tests/terminal-guard-server.test.ts b/tests/terminal-guard-server.test.ts index 721081611..ba596cb80 100644 --- a/tests/terminal-guard-server.test.ts +++ b/tests/terminal-guard-server.test.ts @@ -270,4 +270,51 @@ describe("server terminal guard integration", () => { expect(sends).toBe(2); }); + test("terminal-guard 429 wait longer than the stall budget still succeeds (heartbeats)", async () => { + const stallConfig = { + ...config, + stallTimeoutSec: 1, + providers: { + "claude-se": { + adapter: "anthropic", + baseUrl: "https://example.test", + apiKey: "sk-test", + retryOn429: { attempts: 1, intervalMs: 1_500, respectRetryAfter: false }, + }, + }, + } as unknown as OcxConfig; + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) { + return anthropicSse(firstTurn); + } + if (sends === 2) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + return anthropicSse(continuationTurn); + }) as typeof fetch; + + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "se-claude-opus-4.8", + input: "请检查这个问题并修复代码", + stream: true, + tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }], + }), + }), stallConfig, { model: "", provider: "" }); + + const text = await response.text(); + // A 1.5s continuation backoff under a 1s stall budget must not trip upstream_stall_timeout: + // the wait yields heartbeat events, the replay lands, and the turn completes. + expect(sends).toBe(3); + expect(text).toContain("response.completed"); + expect(text).not.toContain("upstream_stall_timeout"); + }, 5_000); + }); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index 52a1263a3..c023f5f24 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -633,14 +633,18 @@ describe("web-search sidecar native web_search_call emission", () => { let sends = 0; let rotations = 0; let retrySends = 0; + let builds = 0; const retryingAdapter: ProviderAdapter = { name: "mock-retry429", - buildRequest: () => ({ - url: "https://routed.test/v1", - method: "POST", - headers: {}, - body: "{}", - }), + buildRequest: () => { + builds += 1; + return { + url: "https://routed.test/v1", + method: "POST", + headers: {}, + body: "{}", + }; + }, fetchResponse: async () => { sends += 1; if (sends === 1) { @@ -680,8 +684,95 @@ describe("web-search sidecar native web_search_call emission", () => { expect(sends).toBe(2); expect(rotations).toBe(0); expect(retrySends).toBe(1); + // Same-target replay reuses the ONE built request (builder runs once per target sequence). + expect(builds).toBe(1); }); + test("retry wait longer than the stall budget still succeeds (heartbeats feed the watchdog)", async () => { + globalThis.fetch = (() => Promise.resolve(new Response( + 'event: response.completed\ndata: {"type":"response.completed"}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ))) as typeof fetch; + + let sends = 0; + const retryingAdapter: ProviderAdapter = { + name: "mock-retry429", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response("rate limited", { status: 429, headers: { "retry-after": "30" } }); + } + return new Response("{}", { status: 200 }); + }, + async *parseStream() { + yield { type: "text_delta", text: "answer after long backoff" }; + yield { type: "done" }; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter: retryingAdapter, + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.4-mini", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + stallTimeoutSec: 1, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 1_500, maxIntervalMs: 60_000, respectRetryAfter: false }, + }); + const frames = await collectSse(response.body!); + // A 1.5s backoff under a 1s stall budget must not trip upstream_stall_timeout. + expect(sends).toBe(2); + expect(frames.find(f => f.event === "response.completed")).toBeDefined(); + expect(frames.find(f => f.event === "response.failed")).toBeUndefined(); + }, 5_000); + + test("retry wait longer than connectTimeoutMs restarts the header deadline (no 504)", async () => { + globalThis.fetch = (() => Promise.resolve(new Response( + 'event: response.completed\ndata: {"type":"response.completed"}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ))) as typeof fetch; + + let sends = 0; + const retryingAdapter: ProviderAdapter = { + name: "mock-retry429", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + sends += 1; + if (sends === 1) { + return new Response("rate limited", { status: 429, headers: { "retry-after": "30" } }); + } + return new Response("{}", { status: 200 }); + }, + async *parseStream() { + yield { type: "text_delta", text: "answer after deadline restart" }; + yield { type: "done" }; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter: retryingAdapter, + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.4-mini", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + connectTimeoutMs: 100, + retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 150, maxIntervalMs: 60_000, respectRetryAfter: false }, + }); + const frames = await collectSse(response.body!); + // The deliberate backoff must not consume the response-header deadline: a fresh deadline is + // armed after the wait, so the replay gets a new connect budget instead of a 504. + expect(sends).toBe(2); + expect(frames.find(f => f.event === "response.completed")).toBeDefined(); + expect(frames.find(f => f.event === "response.failed")).toBeUndefined(); + }, 5_000); + test("retryOn429 budget is shared across iterations (per request, not per round)", async () => { globalThis.fetch = ((input) => { const url = String(input); From 4e172054f300e6f78f59a0667c2597de39a869ef Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 14:50:58 +0800 Subject: [PATCH 16/29] fix(proxy): clamp heartbeat interval guard; pin emptied-policy opt-in contract --- src/lib/upstream-retry.ts | 6 +++++- tests/config-user-edits.test.ts | 27 +++++++++++++++++++++++++++ tests/upstream-retry.test.ts | 20 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 9eb488209..cfed99a9d 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -83,9 +83,13 @@ export async function* sleepWithHeartbeats( signal?: AbortSignal, heartbeatIntervalMs = 10_000, ): AsyncGenerator<{ type: "heartbeat" }> { + if (ms <= 0) return; + // Guard against a non-positive interval: a zero/negative step would spin the loop forever + // while sleepWithAbort early-returns without ever observing the abort signal. + const stepMs = Math.max(1, heartbeatIntervalMs); let remaining = ms; while (remaining > 0) { - const chunk = Math.min(remaining, heartbeatIntervalMs); + const chunk = Math.min(remaining, stepMs); await sleepWithAbort(chunk, signal); remaining -= chunk; yield { type: "heartbeat" }; diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 2c66c97d3..ea8103b12 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -10,6 +10,7 @@ import { saveConfig, saveConfigPreservingClaudeCode, } from "../src/config"; +import { rateLimitRetryPolicyFor } from "../src/providers/key-failover"; import type { OcxConfig } from "../src/types"; /** @@ -165,6 +166,32 @@ test("an invalid retryOn429 master switch discards the policy instead of enablin expect(live.providers.test.retryOn429).toBeUndefined(); }); +test("a retryOn429 policy degraded to an empty object still resolves as enabled (presence = opt-in)", () => { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + // Every field invalid: the sanitizer drops them all and writes back {}. + retryOn429: { attempts: 0 }, + }, + }, + }); + const live = loadConfig(); + expect(live.providers.test.retryOn429).toEqual({}); + // Object presence is the opt-in contract: an emptied object resolves to the enabled defaults, + // exactly like an explicit `retryOn429: {}` in a hand-written config. + expect(rateLimitRetryPolicyFor(live.providers.test)).toEqual({ + enabled: true, + attempts: 3, + intervalMs: 5_000, + maxIntervalMs: 60_000, + respectRetryAfter: true, + }); +}); + test("invalid retryOn429 values never log the raw value", () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); try { diff --git a/tests/upstream-retry.test.ts b/tests/upstream-retry.test.ts index b1f4d4b65..b2d572baa 100644 --- a/tests/upstream-retry.test.ts +++ b/tests/upstream-retry.test.ts @@ -3,6 +3,7 @@ import { fetchWithResetRetry, isConnectionResetError, retryBackoffDelayMs, + sleepWithHeartbeats, } from "../src/lib/upstream-retry"; function bunResetError(): Error { @@ -60,6 +61,25 @@ describe("isConnectionResetError", () => { }); }); +describe("sleepWithHeartbeats", () => { + test("a non-positive heartbeat interval is clamped instead of spinning forever", async () => { + const events: string[] = []; + for await (const event of sleepWithHeartbeats(3, undefined, 0)) { + events.push(event.type); + } + // 3ms of wait with a clamped 1ms step -> exactly 3 beats, then termination (no spin). + expect(events).toHaveLength(3); + }); + + test("zero wait yields nothing", async () => { + const events: string[] = []; + for await (const event of sleepWithHeartbeats(0, undefined)) { + events.push(event.type); + } + expect(events).toEqual([]); + }); +}); + describe("fetchWithResetRetry", () => { test("retries a Bun-shaped reset and returns the second attempt's response", async () => { silenceWarn(); From 026018155878905ae89be4ff9a7bec7770bb08b9 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Sun, 2 Aug 2026 15:11:54 +0800 Subject: [PATCH 17/29] fix(proxy): normalize NaN heartbeat intervals to the 1ms step --- src/lib/upstream-retry.ts | 5 +++-- tests/upstream-retry.test.ts | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index cfed99a9d..2ba68cb0e 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -85,8 +85,9 @@ export async function* sleepWithHeartbeats( ): AsyncGenerator<{ type: "heartbeat" }> { if (ms <= 0) return; // Guard against a non-positive interval: a zero/negative step would spin the loop forever - // while sleepWithAbort early-returns without ever observing the abort signal. - const stepMs = Math.max(1, heartbeatIntervalMs); + // while sleepWithAbort early-returns without ever observing the abort signal. NaN must be + // normalized too: Math.max(1, NaN) is NaN, which would abort the wait after one beat. + const stepMs = Number.isNaN(heartbeatIntervalMs) ? 1 : Math.max(1, heartbeatIntervalMs); let remaining = ms; while (remaining > 0) { const chunk = Math.min(remaining, stepMs); diff --git a/tests/upstream-retry.test.ts b/tests/upstream-retry.test.ts index b2d572baa..bcb87eaa2 100644 --- a/tests/upstream-retry.test.ts +++ b/tests/upstream-retry.test.ts @@ -71,6 +71,18 @@ describe("sleepWithHeartbeats", () => { expect(events).toHaveLength(3); }); + test("a NaN heartbeat interval waits the full duration instead of aborting after one beat", async () => { + const started = Date.now(); + const events: string[] = []; + for await (const event of sleepWithHeartbeats(120, undefined, Number.NaN)) { + events.push(event.type); + } + // NaN falls back to the 1ms step: the full 120ms wait happens (120 beats), instead of the + // buggy NaN-chunk path that exited after one beat. + expect(events).toHaveLength(120); + expect(Date.now() - started).toBeGreaterThanOrEqual(110); + }); + test("zero wait yields nothing", async () => { const events: string[] = []; for await (const event of sleepWithHeartbeats(0, undefined)) { From 6c02b8f0e621d3699c22d42a595b74152833c3c8 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 00:45:59 +0800 Subject: [PATCH 18/29] fix(proxy): guard adapter builds; bounded 429 body-release; docs scope - Wrap initial/rebuild/terminal buildRequest in abort-guarded try/catch: release body observation, tear down the abort link, abort upstream, and map failures to 400 invalid_request_error (or 499 on client abort); terminal continuations rethrow as in-stream errors. - Add releaseResponseBodyBestEffort and use it at all five 429 body-release sites so a never-settling cancel() cannot block the abort-aware backoff. - Classify terminal-continuation fetch failures as 499 when the upstream signal aborted, not only the client signal. - Document pre-stream-only replay scope, runTurn exclusion, and the request-wide attempts budget in all five locale docs. - Cover build-throw mapping, client-abort-during-build, and never-settling cancel with regression tests. --- .../010_design.md | 4 +- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/images/loop.ts | 7 +- src/lib/upstream-retry.ts | 44 +++++++++ src/providers/xai-transport.ts | 4 +- src/server/responses/core.ts | 92 +++++++++++++------ src/web-search/loop.ts | 7 +- structure/04_transports-and-sidecars.md | 3 +- tests/abort-race.test.ts | 49 ++++++++++ tests/rate-limit-retry.test.ts | 55 +++++++++++ tests/upstream-retry.test.ts | 50 ++++++++++ 15 files changed, 283 insertions(+), 42 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index d34800c01..6067587df 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -82,8 +82,8 @@ existing multi-key failover runs. Default off → zero behavior change for exist shared between concurrent requests (unlike the Kiro 429 pattern). Upstream volume per request: same-key replays add at most `attempts` sends, then multi-key failover adds up to `poolKeys − 1` more (or Anthropic account rotations), so the combined bound is - `attempts + poolKeys` sends — a storm multiplies by that factor per request, not by - `attempts + 1`. + `attempts + poolKeys` sends (pool size = configured `apiKeyPool` length, fixed per request) — + a storm multiplies by that factor per request, not by `attempts + 1`. - Header deadlines: the image/video and web-search bridge loops restart their response-header deadline after each deliberate wait, so backoffs never consume the connect budget and a rate-limit wait is never misattributed as a 504 header timeout. The old deadline is cleared diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 427f73af8..5847dbc64 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -74,7 +74,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `noPenaltyModels?` | `string[]` |存在/周波数ペナルティを拒否するモデル。 | | `parallelToolCalls?` | `boolean` |並列ツール呼び出しを切り替えます。 OpenAI Chat はデフォルトでオンになっています。非チャット アダプターは明示的な `true` でのみアドバタイズします。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` |正確なプレースホルダー ID および欠落している端末 ID に対するダウンストリーム SSE 修復はデフォルトで無効になっています。関数呼び出し ID は決して書き換えられません。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。キー認証の passthrough ワイヤにはキー フェイルオーバーがないため、予算を使い切ると 429 がそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` が `auto` または `none` のみを受け入れるモデル。強制的な選択は格下げされます。 | | `preserveReasoningContentModels?` | `string[]` |チャット履歴に以前のアシスタント `reasoning_content` が必要なモデル。 | | `thinkingToggleModels?` | `string[]` |エフォート ラダーではなく `thinking.enabled` を使用してモデルをチャットします。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 5a702ac3e..1ccf1df16 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -74,7 +74,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `noPenaltyModels?` | `string[]` | presence/frequency penalty를 허용하지 않는 모델입니다. | | `parallelToolCalls?` | `boolean` | 병렬 도구 호출을 켜거나 끕니다. OpenAI Chat은 기본으로 켜져 있고, 비-chat 어댑터는 명시적으로 `true`일 때만 이를 노출합니다. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 기본값이 꺼진 downstream SSE 복구입니다. 정확한 자리표시자 id와 누락된 종료 id를 복구합니다. function-call id는 다시 쓰지 않습니다. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. 키 인증 passthrough 와이어에는 키 장애 조치가 없으므로 예산을 모두 쓰면 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. | | `thinkingToggleModels?` | `string[]` | effort 계층 대신 `thinking.enabled`를 쓰는 chat 모델입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index a924e9356..d740181a1 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -85,7 +85,7 @@ differing backup and rewrites known legacy namespaced selected ids to bare ids. | `noPenaltyModels?` | `string[]` | Models that reject presence/frequency penalties. | | `parallelToolCalls?` | `boolean` | Toggle parallel tool calls. OpenAI Chat defaults on; non-chat adapters advertise only on explicit `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Disabled-by-default downstream SSE repair for exact placeholder ids and missing terminal ids. Function-call ids are never rewritten. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries; on the key-auth passthrough wire there is no key failover, so an exhausted budget surfaces the 429. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | | `thinkingToggleModels?` | `string[]` | Chat models using `thinking.enabled` rather than an effort ladder. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 897725192..149ba807c 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -90,7 +90,7 @@ cross-route credential fallback не существует. Строки API GPT- | `noPenaltyModels?` | `string[]` | Модели, отвергающие penalty presence/frequency. | | `parallelToolCalls?` | `boolean` | Переключатель parallel tool call'ов. Для OpenAI Chat по умолчанию включено; не-chat adapter'ы рекламируют это только при явном `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | По умолчанию выключенная downstream SSE-repair для exact placeholder-id и отсутствующих terminal-id. Function-call id никогда не переписываются. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста; на passthrough-канале с ключевой аутентификацией фейловера ключей нет, поэтому при исчерпании бюджета 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие `thinking.enabled` вместо effort-ladder. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 264a9d8c3..088b5efac 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -74,7 +74,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `noPenaltyModels?` | `string[]` | 会拒绝 presence/frequency penalty 的模型。 | | `parallelToolCalls?` | `boolean` | 切换并行工具调用。OpenAI Chat 默认开启;非 chat 适配器只有显式 `true` 时才会声明支持。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 默认关闭的下游 SSE 修复,用于精确占位 id 和缺失的终止 id。function-call id 永远不会被重写。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应;自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数(总发送次数 = `attempts` + 1),是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算;key 认证的 passthrough 线路上没有 key 故障转移,因此预算耗尽时 429 会原样透出。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型;强制选择会被降级。 | | `preserveReasoningContentModels?` | `string[]` | 需要在聊天历史中保留先前 assistant `reasoning_content` 的模型。 | | `thinkingToggleModels?` | `string[]` | 使用 `thinking.enabled` 而不是 effort 阶梯的 chat 模型。 | diff --git a/src/images/loop.ts b/src/images/loop.ts index 55df17b2f..6f00a904a 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -20,7 +20,7 @@ import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry"; +import { fetchWithResetRetry, releaseResponseBodyBestEffort, sleepWithHeartbeats } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -501,8 +501,9 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } + // Release the body without letting a never-settling cancel() block the abort-aware + // backoff (bounded by the signal and a short timeout). + await releaseResponseBodyBestEffort(prepared.response.body, signal); // The old header deadline must not stay armed across the deliberate wait: clear it // before sleeping so a stale expiry can never race the client-cancel path. headerDeadline.clear(); diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 2ba68cb0e..670778af1 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -71,6 +71,50 @@ export async function sleepWithAbort(ms: number, signal?: AbortSignal): Promise< }); } +/** + * Best-effort, bounded cancellation of a response body before a retry backoff. + * + * The 429 paths release the unread body before waiting so sockets do not accumulate under a + * rate-limit storm, but a never-settling `cancel()` promise must not be able to block the + * abort-aware backoff (client cancel, `maxIntervalMs`, or the cumulative header deadline). + * Cancellation is started and its rejection observed; the await is bounded by `timeoutMs` + * and the abort signal. This mirrors the rotation-path guarantee (release is initiated, not + * awaited forever) while preserving the resource-release intent of the same-target paths. + */ +export async function releaseResponseBodyBestEffort( + body: ReadableStream | null, + signal: AbortSignal | undefined, + timeoutMs = 1_000, +): Promise { + if (!body) return; + if (signal?.aborted) { + void body.cancel().catch(() => {}); + return; + } + const cancel = body.cancel().catch(() => {}); + if (!signal) { + await Promise.race([cancel, new Promise(resolve => setTimeout(resolve, timeoutMs))]); + return; + } + await new Promise(resolve => { + let timer: ReturnType; + const onAbort = () => { + clearTimeout(timer); + resolve(); + }; + timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, timeoutMs); + signal.addEventListener("abort", onAbort, { once: true }); + void cancel.then(() => { + clearTimeout(timer); + signal.removeEventListener("abort", onAbort); + resolve(); + }); + }); +} + /** * Abort-aware sleep that yields an adapter `heartbeat` at least every `heartbeatIntervalMs`. * The Responses bridge treats a returned iterator event as upstream liveness and aborts turns diff --git a/src/providers/xai-transport.ts b/src/providers/xai-transport.ts index b08d12798..1e56506ec 100644 --- a/src/providers/xai-transport.ts +++ b/src/providers/xai-transport.ts @@ -83,7 +83,9 @@ export function deriveXaiConvId(promptCacheKey: string): string { /** * Resolve xAI's runtime transport without mutating persisted config. Conversation/session - * affinity is stable for this resolved transport; request identity is generated per fetch. + * affinity is stable for this resolved transport; request identity is pinned per resolved + * transport (= per logical request until key rotation), so same-target replays and transient + * retries carry the same id while a rotated key gets a fresh one. * Agent, deployment, model-override, turn, mode, and user identity headers are intentionally * omitted because opencodex has no truthful values for the official fields. */ diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 411254fe7..b6fc6c582 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -89,6 +89,7 @@ import { applyUpstreamRecoveryInit, fetchWithResetRetry, fetchWithTransientRetry, + releaseResponseBodyBestEffort, sleepWithHeartbeats, sleepWithAbort, } from "../../lib/upstream-retry"; @@ -1657,8 +1658,9 @@ async function handleResponsesInner( rateLimitRetries += 1; // Release the unread 429 body before the backoff (only the header is needed for the wait). const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. - try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // Release the body without letting a never-settling cancel() block the abort-aware + // backoff (bounded by the abort signal and a short timeout). + await releaseResponseBodyBestEffort(upstreamResponse.body, options.abortSignal); try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -2343,12 +2345,27 @@ async function handleResponsesInner( // builder runs again only after a key/account/adapter rotation, an oauth refresh, or an // image-tier bias change (transportToken bump). `body` is always a serialized string, so // reuse is safe, and releaseBodyObservation is idempotent per build. - const initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); - recordAdapterReasoning(logCtx, initialRequest); - const inputTokenEstimate = typeof initialRequest.usageLog?.inputTokens === "number" - ? initialRequest.usageLog.inputTokens - : undefined; - if (inputTokenEstimate !== undefined) logCtx.usageLogInputTokens = inputTokenEstimate; + let initialRequest: AdapterRequest | undefined; + let inputTokenEstimate: number | undefined; + try { + initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + recordAdapterReasoning(logCtx, initialRequest); + inputTokenEstimate = typeof initialRequest.usageLog?.inputTokens === "number" + ? initialRequest.usageLog.inputTokens + : undefined; + if (inputTokenEstimate !== undefined) logCtx.usageLogInputTokens = inputTokenEstimate; + } catch (err) { + // A throwing buildRequest never returned a request; if a post-build step threw, release + // the serialized-body observation (idempotent) so the translator budget is not leaked. + // The build runs after linkAbortSignal, so a failure must also tear the link down and + // abort the upstream controller instead of escaping handleResponses unmapped. + initialRequest?.releaseBodyObservation?.(); + cleanupUpstreamAbort(); + upstream.abort(); + if (options.abortSignal?.aborted) return clientCancelledResponse(); + const msg = err instanceof Error ? err.message : String(err); + return formatErrorResponse(400, "invalid_request_error", redactSecretString(msg)); + } let sameTargetRequest: AdapterRequest | undefined = initialRequest; let sameTargetParsed: OcxParsedRequest | undefined = parsed; let sameTargetToken = 0; @@ -2413,12 +2430,23 @@ async function handleResponsesInner( // Same target (key/adapter/parsed/tier unchanged): replay the exact cached request. retryRequest = sameTargetRequest; } else { - retryRequest = await activeAdapter.buildRequest(parsed, { - headers: selectedForwardHeaders, - translatorBudget, - ...(imageTierBias > 0 ? { imageTierBias } : {}), - }); - recordAdapterReasoning(logCtx, retryRequest); + try { + retryRequest = await activeAdapter.buildRequest(parsed, { + headers: selectedForwardHeaders, + translatorBudget, + ...(imageTierBias > 0 ? { imageTierBias } : {}), + }); + recordAdapterReasoning(logCtx, retryRequest); + } catch (err) { + // A rotated/rebuilt adapter build failure is a request-shaping error, not an + // upstream connect failure: tear the abort link down and map it as 400 (no 413 + // translator-budget mapping here — that stays with parseRequest/buildToolBridgeMaps). + cleanupUpstreamAbort(); + upstream.abort(); + if (options.abortSignal?.aborted) return { failed: clientCancelledResponse() }; + const msg = err instanceof Error ? err.message : String(err); + return { failed: formatErrorResponse(400, "invalid_request_error", redactSecretString(msg)) }; + } sameTargetRequest = retryRequest; sameTargetParsed = parsed; sameTargetToken = transportToken; @@ -2504,8 +2532,9 @@ async function handleResponsesInner( // wait, and under a 429 storm the sockets would otherwise accumulate for the whole // configured interval (same pattern as the key-failover branch below). const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. - try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // Release the body without letting a never-settling cancel() block the abort-aware + // backoff (bounded by the abort signal and a short timeout). + await releaseResponseBodyBestEffort(upstreamResponse.body, options.abortSignal); try { await sleepWithAbort( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -2660,17 +2689,25 @@ async function handleResponsesInner( * deterministic for the same parsed request (tests assert byte-identical replays). */ const fetchContinuation = async (replay = false): Promise => { - let continuationRequest: AdapterRequest; + let continuationRequest: AdapterRequest | undefined; if (sameTargetRequest !== undefined && sameTargetParsed === nextParsed && sameTargetToken === transportToken) { // Same target (key/adapter/parsed/tier unchanged): replay the exact cached request. continuationRequest = sameTargetRequest; } else { - continuationRequest = await activeAdapter.buildRequest(nextParsed, { - headers: selectedForwardHeaders, - translatorBudget, - ...(imageTierBias > 0 ? { imageTierBias } : {}), - }); - recordAdapterReasoning(logCtx, continuationRequest); + try { + continuationRequest = await activeAdapter.buildRequest(nextParsed, { + headers: selectedForwardHeaders, + translatorBudget, + ...(imageTierBias > 0 ? { imageTierBias } : {}), + }); + recordAdapterReasoning(logCtx, continuationRequest); + } catch (err) { + // The main body is already streaming, so there is no HTTP error surface: release + // any partial body observation and surface the failure as an in-stream error via + // the outer catch (no upstream.abort() — that would kill the live body stream). + continuationRequest?.releaseBodyObservation?.(); + throw err; + } sameTargetRequest = continuationRequest; sameTargetParsed = nextParsed; sameTargetToken = transportToken; @@ -2714,7 +2751,7 @@ async function handleResponsesInner( try { response = await fetchContinuation(); } catch (error) { - if (options.abortSignal?.aborted) { + if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; } else { yield { type: "error", message: `Provider continuation failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -2733,8 +2770,9 @@ async function handleResponsesInner( rateLimitRetries += 1; // Release the unread 429 body before the backoff (only the header is needed for the wait). const retryAfterHeader = response.headers.get("retry-after"); - // AWAIT the cancellation so the resource-release guarantee is real, not best-effort. - try { await response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + // Release the body without letting a never-settling cancel() block the abort-aware + // backoff (bounded by the upstream signal and a short timeout). + await releaseResponseBodyBestEffort(response.body, upstream.signal); try { yield* sleepWithHeartbeats( rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), @@ -2761,7 +2799,7 @@ async function handleResponsesInner( try { response = await fetchContinuation(true); } catch (error) { - if (options.abortSignal?.aborted) { + if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; } else { yield { type: "error", message: `Provider continuation failed: ${error instanceof Error ? error.message : String(error)}` }; diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index a6fa4873f..5bd96fab6 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -8,7 +8,7 @@ import { runAnthropicWebSearch } from "./anthropic-executor"; import { clearableDeadline } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry"; +import { fetchWithResetRetry, releaseResponseBodyBestEffort, sleepWithHeartbeats } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -404,8 +404,9 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } + // Release the body without letting a never-settling cancel() block the abort-aware + // backoff (bounded by the signal and a short timeout). + await releaseResponseBodyBestEffort(prepared.response.body, signal); // The old header deadline must not stay armed across the deliberate wait: clear it // before sleeping so a stale expiry can never race the client-cancel path. headerDeadline.clear(); diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index b7aa94875..dde87c628 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -232,7 +232,8 @@ failover). Codex never retries 429 client-side (openai/codex#30471), so this is defense for those providers; the final 429 still carries `Retry-After` for clients that honor it. Concurrent requests each honor their own policy — there is no process-wide shared cooldown (unlike the Kiro pattern), so a rate-limit storm multiplies upstream volume by at most -`attempts + poolKeys` per request (same-key replays, then failover keys). Every surface +`attempts + poolKeys` per request (same-key replays, then failover keys; the pool size is the +operator-configured `apiKeyPool` length, fixed for the duration of the request). Every surface releases (and awaits the cancellation of) the unread 429 body before the backoff, records the `rate-limit-429` recovery kind on replay sends, and the bridge loops clear the old response-header deadline before the wait and start a fresh one afterward — client cancellation diff --git a/tests/abort-race.test.ts b/tests/abort-race.test.ts index 0e8458db7..6c0d351f3 100644 --- a/tests/abort-race.test.ts +++ b/tests/abort-race.test.ts @@ -190,4 +190,53 @@ describe("Responses abort guards", () => { process.off("unhandledRejection", onUnhandledRejection); } }); + + test("a throwing buildRequest is mapped to 400 invalid_request_error, not an unhandled rejection", async () => { + const unhandledRejections: unknown[] = []; + const onUnhandledRejection = (reason: unknown) => { unhandledRejections.push(reason); }; + process.on("unhandledRejection", onUnhandledRejection); + try { + adapterFactory = provider => ({ + name: "test-throw-build", + buildRequest: () => { throw new Error("fixture build failure"); }, + async *parseStream(): AsyncGenerator { + yield { type: "error", message: "unreachable" }; + }, + }); + + const response = await post("test-throw-build", false); + const body = await response.json() as { error?: { code?: string; message?: string } }; + + expect(response.status).toBe(400); + expect(body.error?.code).toBe("invalid_request_error"); + expect(body.error?.message).toContain("fixture build failure"); + expect(unhandledRejections).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandledRejection); + } + }); + + test("a client abort during buildRequest surfaces 499 client_cancelled instead of a 400", async () => { + const clientAbort = new AbortController(); + adapterFactory = provider => ({ + name: "test-abort-build", + buildRequest: () => { + clientAbort.abort(new DOMException("client disconnected", "AbortError")); + throw new Error("build interrupted by client disconnect"); + }, + async *parseStream(): AsyncGenerator { + yield { type: "error", message: "unreachable" }; + }, + }); + + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "fixture/model", input: "hello", stream: false }), + }), config("test-abort-build"), { model: "", provider: "" }, { abortSignal: clientAbort.signal }); + const body = await response.json() as { error?: { code?: string } }; + + expect(response.status).toBe(499); + expect(body.error?.code).toBe("client_cancelled"); + }); }); diff --git a/tests/rate-limit-retry.test.ts b/tests/rate-limit-retry.test.ts index 6db97769c..244d244f1 100644 --- a/tests/rate-limit-retry.test.ts +++ b/tests/rate-limit-retry.test.ts @@ -171,4 +171,59 @@ describe("retry loop client-abort handling", () => { const body = await response.json() as { error?: { code?: string } }; expect(body.error?.code).toBe("client_cancelled"); }); + + test("a never-settling 429 body cancel() cannot block the abort-aware backoff", async () => { + let sends = 0; + let cancelInitiated = false; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://llmapi.blsc.cn/chat/completions") { + sends += 1; + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(JSON.stringify({ error: { message: "rate limited" } }))); + controller.close(); + }, + cancel() { + cancelInitiated = true; + // Never settles: the release must be bounded or the retry loop hangs here. + return new Promise(() => {}); + }, + }), { status: 429, headers: { "content-type": "application/json" } }); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config = { + port: 0, + defaultProvider: "blsc", + providers: { + blsc: { + adapter: "openai-chat", + baseUrl: "https://llmapi.blsc.cn", + authMode: "key", + apiKey: "key-alpha-000111222333", + retryOn429: { attempts: 3, intervalMs: 30_000, respectRetryAfter: false }, + }, + }, + } as OcxConfig; + + const abort = new AbortController(); + const pending = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "blsc/DeepSeek-V4-Flash", input: "hello", stream: false }), + }), config, { model: "blsc/DeepSeek-V4-Flash", provider: "blsc" }, { abortSignal: abort.signal }); + + for (let i = 0; i < 100 && sends === 0; i += 1) await Bun.sleep(10); + expect(sends).toBe(1); + + abort.abort(new DOMException("client disconnected", "AbortError")); + const started = Date.now(); + const response = await pending; + expect(Date.now() - started).toBeLessThan(2_000); + expect(response.status).toBe(499); + expect(sends).toBe(1); + expect(cancelInitiated).toBe(true); + }); }); diff --git a/tests/upstream-retry.test.ts b/tests/upstream-retry.test.ts index bcb87eaa2..bd11bc9cf 100644 --- a/tests/upstream-retry.test.ts +++ b/tests/upstream-retry.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { fetchWithResetRetry, isConnectionResetError, + releaseResponseBodyBestEffort, retryBackoffDelayMs, sleepWithHeartbeats, } from "../src/lib/upstream-retry"; @@ -92,6 +93,55 @@ describe("sleepWithHeartbeats", () => { }); }); +describe("releaseResponseBodyBestEffort", () => { + test("a never-settling cancel() does not block past the bounded timeout", async () => { + const signal = new AbortController().signal; + const body = new ReadableStream({ + cancel() { + // Never settles — the release must still be bounded. + return new Promise(() => {}); + }, + }); + const started = Date.now(); + await releaseResponseBodyBestEffort(body, signal, 120); + const elapsed = Date.now() - started; + expect(elapsed).toBeGreaterThanOrEqual(100); + expect(elapsed).toBeLessThan(1_000); + }); + + test("a never-settling cancel() resolves immediately when the signal aborts", async () => { + const controller = new AbortController(); + const body = new ReadableStream({ + cancel() { + return new Promise(() => {}); + }, + }); + const pending = releaseResponseBodyBestEffort(body, controller.signal, 60_000); + controller.abort(new DOMException("client disconnected", "AbortError")); + const started = Date.now(); + await pending; + expect(Date.now() - started).toBeLessThan(500); + }); + + test("an already-aborted signal initiates cancellation without awaiting it", async () => { + const controller = new AbortController(); + controller.abort(); + let cancelInitiated = false; + const body = new ReadableStream({ + cancel() { + cancelInitiated = true; + return new Promise(() => {}); + }, + }); + await releaseResponseBodyBestEffort(body, controller.signal, 60_000); + expect(cancelInitiated).toBe(true); + }); + + test("null body is a no-op", async () => { + await expect(releaseResponseBodyBestEffort(null, new AbortController().signal, 10)).resolves.toBeUndefined(); + }); +}); + describe("fetchWithResetRetry", () => { test("retries a Bun-shaped reset and returns the second attempt's response", async () => { silenceWarn(); From e5021734cbe26b07a8b2c6fa3ddf082554a6499c Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 20:07:01 +0800 Subject: [PATCH 19/29] fix(config): single-source retryOn429 bounds; validate at the management write boundary CodeRabbit nitpick: extract the duplicated retryOn429 field bounds into one strict shared schema used by providerConfigSchema, the load-time sanitizer (checks derived from the schema shape), and a new retryOn429PolicyConfigError called from providerManagementConfigError so /api/providers writes reject invalid values and unknown keys before persisting. Error text never echoes values and secret-shaped unknown field names are redacted. --- src/config.ts | 65 ++++++++++++++------ src/server/auth-cors.ts | 3 + tests/management-provider-validation.test.ts | 36 +++++++++++ 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/src/config.ts b/src/config.ts index 88ec15c9e..9211fe598 100644 --- a/src/config.ts +++ b/src/config.ts @@ -551,6 +551,23 @@ export function reconcileConfigWarningMemos(generation: number): number { return removed; } +/** + * Bounds for the opt-in same-target 429 wait-and-retry policy. Single source of truth + * shared by the config schema, the load-time sanitizer, and the management write + * boundary. Strict, so an unknown key is rejected at every validation boundary instead + * of being silently ignored (the load-time sanitizer still degrades unknown keys with a + * warning before schema validation, so hand-edited configs keep loading). + */ +const retryOn429PolicySchema = z.object({ + enabled: z.boolean().optional(), + attempts: z.number().int().min(1).max(20).optional(), + intervalMs: z.number().int().min(100).max(600_000).optional(), + // The effective cap for a single wait is MAX_COOLDOWN_MS (10 min) in key-failover.ts; + // larger configured values would be dead config. + maxIntervalMs: z.number().int().min(100).max(600_000).optional(), + respectRetryAfter: z.boolean().optional(), +}).strict(); + const providerConfigSchema = z.object({ adapter: z.string().min(1), baseUrl: z.string().min(1), @@ -563,15 +580,7 @@ const providerConfigSchema = z.object({ supportsServiceTier: z.boolean().optional(), preserveResponsesReasoningContent: z.boolean().optional(), allowPrivateNetwork: z.boolean().optional(), - retryOn429: z.object({ - enabled: z.boolean().optional(), - attempts: z.number().int().min(1).max(20).optional(), - intervalMs: z.number().int().min(100).max(600_000).optional(), - // The effective cap for a single wait is MAX_COOLDOWN_MS (10 min) in key-failover.ts; - // larger configured values would be dead config. - maxIntervalMs: z.number().int().min(100).max(600_000).optional(), - respectRetryAfter: z.boolean().optional(), - }).optional(), + retryOn429: retryOn429PolicySchema.optional(), codexAccountMode: z.enum(["pool", "direct"]).optional(), responsesItemIdRepair: z.object({ message: z.array(z.string().min(1)).optional(), @@ -1337,22 +1346,18 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.enabled (${typeof policyRecord.enabled}) is invalid — ignoring the whole policy`); continue; } - const fields: Array<[string, (value: unknown) => boolean]> = [ - ["enabled", value => typeof value === "boolean"], - ["attempts", value => typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 20], - ["intervalMs", value => typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 600_000], - ["maxIntervalMs", value => typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 600_000], - ["respectRetryAfter", value => typeof value === "boolean"], - ]; + // Field checks derive from the shared policy schema so the bounds cannot drift + // between the load-time sanitizer, the config schema, and the write boundary. + const policyShape = retryOn429PolicySchema.shape; const cleaned: Record = {}; - for (const [key, isValid] of fields) { + for (const [key, fieldSchema] of Object.entries(policyShape)) { const value = policyRecord[key]; if (value === undefined) continue; - if (isValid(value)) cleaned[key] = value; + if (fieldSchema.safeParse(value).success) cleaned[key] = value; // Log only the received type, never the value (provider config can hold secrets). else console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.${key} (${typeof value}) is invalid — ignoring the field`); } - const knownKeys = new Set(fields.map(([key]) => key)); + const knownKeys = new Set(Object.keys(policyShape)); for (const key of Object.keys(policyRecord)) { if (!knownKeys.has(key)) { // Redact the field NAME before logging: a malformed hand-edit can place a secret in a @@ -1366,6 +1371,28 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { } } +/** + * Management write-boundary validation for `retryOn429` (fail closed). Unlike the + * lenient load-time sanitizer, invalid values and unknown keys are rejected outright so + * a POST/PATCH cannot persist a policy the proxy would then silently degrade. Reuses the + * shared policy schema. Never echoes values, and secret-shaped unknown field names are + * redacted (a malformed write can place a secret in a property name). + */ +export function retryOn429PolicyConfigError(policy: unknown): string | null { + if (policy === undefined) return null; + const result = retryOn429PolicySchema.safeParse(policy); + if (result.success) return null; + const first = result.error.issues[0]; + if (!first) return "retryOn429 is invalid"; + if (first.code === "unrecognized_keys") { + const names = first.keys.map(key => JSON.stringify(redactSecretString(key))).join(", "); + return `retryOn429 has unrecognized field${first.keys.length > 1 ? "s" : ""}: ${names}`; + } + if (first.path.length === 0) return `retryOn429 is invalid (${first.message})`; + const field = String(first.path[first.path.length - 1]); + return `retryOn429.${field} is invalid (${first.message})`; +} + /** * Companion to {@link warnDegradedStreamMode} for a blank persisted `hostname`. The bind * falls back to loopback, which is the safe direction but not what the file asked for — diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index 79b02c329..dc095d466 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -11,6 +11,7 @@ import { providerBaseUrlConfigError, providerHeadersConfigError, reasoningSummaryDeliveryRecordConfigError, + retryOn429PolicyConfigError, } from "../config"; import { providerDestinationConfigError } from "../lib/destination-policy"; import { getProviderRegistryEntry, providerCodexAccountMode, providerMatchesRegistryTransport, registryEntryForProviderDestination } from "../providers/registry"; @@ -420,6 +421,8 @@ export function providerManagementConfigError(name: unknown, provider: unknown): if (destinationError) return `provider ${name} ${destinationError}`; const headersError = providerHeadersConfigError(typed.headers); if (headersError) return `provider ${name} ${headersError}`; + const retryOn429Error = retryOn429PolicyConfigError(raw.retryOn429); + if (retryOn429Error) return `provider ${name} ${retryOn429Error}`; const apiKeyTransportError = apiKeyTransportConfigError(typed); if (apiKeyTransportError) return `provider ${name} ${apiKeyTransportError}`; const maxInputError = positiveIntegerRecordConfigError(raw.modelMaxInputTokens, "modelMaxInputTokens"); diff --git a/tests/management-provider-validation.test.ts b/tests/management-provider-validation.test.ts index 7f4d9e67e..77ed48206 100644 --- a/tests/management-provider-validation.test.ts +++ b/tests/management-provider-validation.test.ts @@ -192,6 +192,42 @@ describe("provider management validation", () => { })).toContain("not supported on forward-auth"); }); + test("provider management validates retryOn429 bounds and unknown keys", () => { + const base = { adapter: "openai-chat", baseUrl: "https://api.openai.com/v1" }; + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: { enabled: true, attempts: 3, intervalMs: 1_000, maxIntervalMs: 5_000, respectRetryAfter: false }, + })).toBeNull(); + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: { attempts: 0 }, + })).toContain("retryOn429.attempts is invalid"); + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: { attempts: 21 }, + })).toContain("retryOn429.attempts is invalid"); + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: { intervalMs: "fast" }, + })).toContain("retryOn429.intervalMs is invalid"); + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: { attempt: 3 }, + })).toContain("retryOn429 has unrecognized field"); + expect(providerManagementConfigError("custom", { + ...base, + retryOn429: "enabled", + })).toContain("retryOn429 is invalid"); + // A secret-shaped unknown field name must be redacted in the error, never echoed. + const secretError = providerManagementConfigError("custom", { + ...base, + retryOn429: { "sk-super-secret-9876": true }, + })!; + expect(secretError).toContain("retryOn429 has unrecognized field"); + expect(secretError).not.toContain("sk-super-secret-9876"); + expect(secretError).toContain("[REDACTED]"); + }); + test("provider discovery status is additive and omitted before an attempt", async () => { markProviderDiscoveryFailed("auth-broken", { reason: "http", httpStatus: 401 }); try { From 73a5b9575100241daacf2f6fd73d36c3d635f717 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 20:14:39 +0800 Subject: [PATCH 20/29] fix(gui): localize attempt recovery kinds in the logs detail dialog CodeRabbit finding: LogDetailDialog rendered AttemptRecoveryKind values (rate-limit-429, anthropic-oauth-429, ...) directly. Map every recovery kind to an i18n key via RECOVERY_KIND_KEYS and add translations for all seven kinds in every GUI locale. --- gui/src/i18n/de.ts | 7 +++++++ gui/src/i18n/en.ts | 7 +++++++ gui/src/i18n/ja.ts | 7 +++++++ gui/src/i18n/ko.ts | 7 +++++++ gui/src/i18n/ru.ts | 7 +++++++ gui/src/i18n/zh.ts | 7 +++++++ gui/src/pages/Logs.tsx | 18 +++++++++++++++++- 7 files changed, 59 insertions(+), 1 deletion(-) diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 6c2985f0d..4c2998468 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -549,6 +549,13 @@ export const de: Record = { "logs.detail.attempt.reason": "Ergebnis / Grund", "logs.detail.attempt.completed": "Abgeschlossen", "logs.detail.attempt.e2eNote": "Tok/s auf oberster Ebene ist Ende-zu-Ende; jeder Versuch nutzt seine eigene Dauer.", + "logs.detail.attempt.recovery.transient5xx": "Vorübergehender 5xx-Fehler", + "logs.detail.attempt.recovery.connectionReset": "Verbindung zurückgesetzt", + "logs.detail.attempt.recovery.oauth401": "OAuth-Neuanmeldung", + "logs.detail.attempt.recovery.key429": "Schlüssel ratenbegrenzt (429)", + "logs.detail.attempt.recovery.rateLimit429": "Ratenbegrenzt (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth ratenbegrenzt (429)", + "logs.detail.attempt.recovery.image413": "Bildnutzlast zu groß (413)", "logs.detail.reason.usage_missing": "Nutzung wurde nicht gemeldet.", "logs.detail.reason.usage_unsupported": "Dieser Anbieter meldet keine Nutzung.", "logs.detail.reason.output_missing": "Es wurden keine positiven Ausgabe-Tokens gemeldet.", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 0deebfa22..6788c496e 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -574,6 +574,13 @@ export const en = { "logs.detail.attempt.reason": "Result / reason", "logs.detail.attempt.completed": "Completed", "logs.detail.attempt.e2eNote": "Top-level tok/s is end-to-end; each attempt uses its own duration.", + "logs.detail.attempt.recovery.transient5xx": "Transient 5xx", + "logs.detail.attempt.recovery.connectionReset": "Connection reset", + "logs.detail.attempt.recovery.oauth401": "OAuth re-authentication", + "logs.detail.attempt.recovery.key429": "Key rate-limited (429)", + "logs.detail.attempt.recovery.rateLimit429": "Rate-limited (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth rate-limited (429)", + "logs.detail.attempt.recovery.image413": "Image payload too large (413)", "logs.detail.reason.usage_missing": "Usage was not reported.", "logs.detail.reason.usage_unsupported": "This provider does not report usage.", "logs.detail.reason.output_missing": "No positive output token count was reported.", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index d48982350..fe680ac64 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -534,6 +534,13 @@ export const ja: Record = { "logs.detail.attempt.reason": "結果 / 理由", "logs.detail.attempt.completed": "完了", "logs.detail.attempt.e2eNote": "トップレベルの tok/s はエンドツーエンドです; 各試行は自身の所要時間を使います。", + "logs.detail.attempt.recovery.transient5xx": "一時的な5xxエラー", + "logs.detail.attempt.recovery.connectionReset": "接続がリセットされました", + "logs.detail.attempt.recovery.oauth401": "OAuth 再認証", + "logs.detail.attempt.recovery.key429": "キーがレート制限 (429)", + "logs.detail.attempt.recovery.rateLimit429": "レート制限 (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth レート制限 (429)", + "logs.detail.attempt.recovery.image413": "画像ペイロードが大きすぎます (413)", "logs.detail.reason.usage_missing": "使用量が報告されませんでした。", "logs.detail.reason.usage_unsupported": "このプロバイダーは使用量を報告しません。", "logs.detail.reason.output_missing": "正の出力トークン数が報告されませんでした。", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index ffd59b5a5..fe92d8be2 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -568,6 +568,13 @@ export const ko: Record = { "logs.detail.attempt.reason": "결과 / 사유", "logs.detail.attempt.completed": "완료", "logs.detail.attempt.e2eNote": "상위 tok/s는 전체 요청 기준이며 각 시도는 자체 소요 시간을 사용합니다.", + "logs.detail.attempt.recovery.transient5xx": "일시적 5xx 오류", + "logs.detail.attempt.recovery.connectionReset": "연결이 재설정됨", + "logs.detail.attempt.recovery.oauth401": "OAuth 재인증", + "logs.detail.attempt.recovery.key429": "키 요금 제한 (429)", + "logs.detail.attempt.recovery.rateLimit429": "요금 제한 (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth 요금 제한 (429)", + "logs.detail.attempt.recovery.image413": "이미지 페이로드가 너무 큼 (413)", "logs.detail.reason.usage_missing": "usage가 보고되지 않았습니다.", "logs.detail.reason.usage_unsupported": "이 프로바이더는 usage 보고를 지원하지 않습니다.", "logs.detail.reason.output_missing": "양수 출력 토큰 수가 보고되지 않았습니다.", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index ee876e25d..8d296a592 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -566,6 +566,13 @@ export const ru: Record = { "logs.detail.attempt.reason": "Результат / причина", "logs.detail.attempt.completed": "Завершено", "logs.detail.attempt.e2eNote": "Общий tok/s — сквозной показатель; для каждой попытки используется её собственная длительность.", + "logs.detail.attempt.recovery.transient5xx": "Временная ошибка 5xx", + "logs.detail.attempt.recovery.connectionReset": "Соединение сброшено", + "logs.detail.attempt.recovery.oauth401": "Повторная авторизация OAuth", + "logs.detail.attempt.recovery.key429": "Ключ ограничен (429)", + "logs.detail.attempt.recovery.rateLimit429": "Ограничение частоты запросов (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth ограничен (429)", + "logs.detail.attempt.recovery.image413": "Слишком большой размер изображения (413)", "logs.detail.reason.usage_missing": "Данные об использовании не были сообщены.", "logs.detail.reason.usage_unsupported": "Этот провайдер не сообщает данные об использовании.", "logs.detail.reason.output_missing": "Положительное число выходных токенов не было сообщено.", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 721f65b06..0d8f899b0 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -561,6 +561,13 @@ export const zh: Record = { "logs.detail.attempt.reason": "结果 / 原因", "logs.detail.attempt.completed": "已完成", "logs.detail.attempt.e2eNote": "顶层 tok/s 为端到端值;每次尝试使用各自耗时。", + "logs.detail.attempt.recovery.transient5xx": "临时 5xx 错误", + "logs.detail.attempt.recovery.connectionReset": "连接已重置", + "logs.detail.attempt.recovery.oauth401": "OAuth 重新认证", + "logs.detail.attempt.recovery.key429": "密钥被限流 (429)", + "logs.detail.attempt.recovery.rateLimit429": "被限流 (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth 被限流 (429)", + "logs.detail.attempt.recovery.image413": "图片载荷过大 (413)", "logs.detail.reason.usage_missing": "未上报 usage。", "logs.detail.reason.usage_unsupported": "该提供方不支持上报 usage。", "logs.detail.reason.output_missing": "未上报正数输出 token。", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index de814e7e5..17584d7a5 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -288,6 +288,16 @@ const ESTIMATE_REASON_KEYS = { expected_price_overlay: "logs.detail.estimate.expected_price_overlay", } as const satisfies Record; +const RECOVERY_KIND_KEYS = { + "transient-5xx": "logs.detail.attempt.recovery.transient5xx", + "connection-reset": "logs.detail.attempt.recovery.connectionReset", + "oauth-401": "logs.detail.attempt.recovery.oauth401", + "key-429": "logs.detail.attempt.recovery.key429", + "rate-limit-429": "logs.detail.attempt.recovery.rateLimit429", + "anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429", + "image-413": "logs.detail.attempt.recovery.image413", +} as const satisfies Record; + function metricReasonKey(reason: MetricUnavailableReason) { return METRIC_REASON_KEYS[reason]; } @@ -296,6 +306,10 @@ function estimateReasonKey(reason: CostEstimateReason) { return ESTIMATE_REASON_KEYS[reason]; } +function recoveryKindKey(kind: AttemptRecoveryKind) { + return RECOVERY_KIND_KEYS[kind]; +} + function verificationKey(status: MatchedPriceInfo["status"]): "logs.detail.verification.verified" | "logs.detail.verification.derived" { return status === "verified" ? "logs.detail.verification.verified" : "logs.detail.verification.derived"; } @@ -962,7 +976,9 @@ function LogDetailDialog({ const attemptReasoningWire = reasoningWireLabel(attempt); const matched = attemptCost?.kind === "value" ? attemptCost.estimate.price : undefined; const reason = attempt.errorCode - ?? (attempt.recoveryKinds.length ? attempt.recoveryKinds.join(", ") : undefined) + ?? (attempt.recoveryKinds.length + ? attempt.recoveryKinds.map(kind => t(recoveryKindKey(kind))).join(", ") + : undefined) ?? (attemptCost?.kind === "unavailable" ? t(metricReasonKey(attemptCost.reason)) : t("logs.detail.attempt.completed")); return ( From 22ac86852aac873264ebda73209c6c5dc7b9c470 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 20:24:18 +0800 Subject: [PATCH 21/29] fix(proxy): narrow request captures, couple cache invalidation, harden sanitizer/error paths CodeRabbit round on the merge head: - Capture initialRequest and continuationRequest in consts after their build blocks so fetch callbacks read narrowed values (let unions kept undefined). - Replace the seven bare transportToken bumps with invalidateSameTargetRequest() so cache invalidation is structurally coupled to credential/adapter mutations. - Drop a retryOn429 policy whose every supplied field was invalid instead of persisting {} (which would opt IN to retries with defaults); keep an intentionally empty policy. - Redact + JSON-escape the provider name in the retryOn429 management error path; regression tests for both behaviors. --- src/config.ts | 12 +++- src/server/auth-cors.ts | 7 ++- src/server/responses/core.ts | 60 ++++++++++++-------- tests/config-user-edits.test.ts | 27 +++++++-- tests/management-provider-validation.test.ts | 8 +++ 5 files changed, 84 insertions(+), 30 deletions(-) diff --git a/src/config.ts b/src/config.ts index 9211fe598..81e4beee9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1349,6 +1349,7 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { // Field checks derive from the shared policy schema so the bounds cannot drift // between the load-time sanitizer, the config schema, and the write boundary. const policyShape = retryOn429PolicySchema.shape; + const hadPolicyEntries = Object.keys(policyRecord).length > 0; const cleaned: Record = {}; for (const [key, fieldSchema] of Object.entries(policyShape)) { const value = policyRecord[key]; @@ -1367,7 +1368,16 @@ function sanitizeRetryOn429ForLoad(parsed: unknown): void { console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429.${JSON.stringify(redactSecretString(key))} is not a recognized field — ignoring it`); } } - p.retryOn429 = cleaned; + if (hadPolicyEntries && Object.keys(cleaned).length === 0) { + // Every supplied field was invalid: drop the whole policy. Persisting `{}` here would + // opt IN to retries with defaults, which is the opposite of what a malformed + // disable-oriented edit (`retryOn429: { enabled: "false" }`, `attempts: 0`) asked for. + delete p.retryOn429; + console.warn(`⚠️ config.json providers.${safeProviderName}.retryOn429 has no valid fields left — removing the policy (an empty policy would enable retries with defaults)`); + } else { + // Preserve an intentionally empty `retryOn429: {}` (presence = opt-in with defaults). + p.retryOn429 = cleaned; + } } } diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index dc095d466..3fe1a6c6a 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -14,6 +14,7 @@ import { retryOn429PolicyConfigError, } from "../config"; import { providerDestinationConfigError } from "../lib/destination-policy"; +import { redactSecretString } from "../lib/redact"; import { getProviderRegistryEntry, providerCodexAccountMode, providerMatchesRegistryTransport, registryEntryForProviderDestination } from "../providers/registry"; import { providerConfigSeed } from "../providers/derive"; import type { OcxConfig, OcxProviderConfig } from "../types"; @@ -422,7 +423,11 @@ export function providerManagementConfigError(name: unknown, provider: unknown): const headersError = providerHeadersConfigError(typed.headers); if (headersError) return `provider ${name} ${headersError}`; const retryOn429Error = retryOn429PolicyConfigError(raw.retryOn429); - if (retryOn429Error) return `provider ${name} ${retryOn429Error}`; + if (retryOn429Error) { + // The provider name is caller-controlled and can be token-shaped; redact and JSON-escape + // it before it reaches the management API response. + return `provider ${JSON.stringify(redactSecretString(name))} ${retryOn429Error}`; + } const apiKeyTransportError = apiKeyTransportConfigError(typed); if (apiKeyTransportError) return `provider ${name} ${apiKeyTransportError}`; const maxInputError = positiveIntegerRecordConfigError(raw.modelMaxInputTokens, "modelMaxInputTokens"); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index f1fecd473..d296e64b6 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2442,15 +2442,23 @@ async function handleResponsesInner( const msg = err instanceof Error ? err.message : String(err); return formatErrorResponse(400, "invalid_request_error", redactSecretString(msg)); } - let sameTargetRequest: AdapterRequest | undefined = initialRequest; + // The catch path above always returns, so the request is definitely assigned here. + // Capture it in a const so the fetch callbacks read a narrowed, immutable value + // (TypeScript drops narrowing for a `let` captured by a nested function). + const builtInitialRequest = initialRequest; + let sameTargetRequest: AdapterRequest | undefined = builtInitialRequest; let sameTargetParsed: OcxParsedRequest | undefined = parsed; let sameTargetToken = 0; let transportToken = 0; + // Invalidate the same-target request cache. Every credential/adapter/parsed mutation MUST + // go through here: the cache keys on `parsed` REFERENCE identity, so an in-place mutation + // is invisible to it and a missed bump would replay a request built with a stale key. + const invalidateSameTargetRequest = (): void => { transportToken += 1; }; let upstreamResponse: Response; try { if (activeAdapter.fetchResponse) { noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate); - upstreamResponse = await activeAdapter.fetchResponse(initialRequest, { + upstreamResponse = await activeAdapter.fetchResponse(builtInitialRequest, { abortSignal: upstream.signal, timeoutMs: connectMs, stream: parsed.stream, @@ -2459,13 +2467,13 @@ async function handleResponsesInner( upstreamResponse = await fetchWithResetRetry( recovery => { noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate, recovery); - return fetchWithHeaderTimeout(initialRequest.url, applyUpstreamRecoveryInit({ - method: initialRequest.method, - headers: initialRequest.headers, - body: initialRequest.body, + return fetchWithHeaderTimeout(builtInitialRequest.url, applyUpstreamRecoveryInit({ + method: builtInitialRequest.method, + headers: builtInitialRequest.headers, + body: builtInitialRequest.body, }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)); }, - { abortSignal: upstream.signal, label: safeHostLabel(initialRequest.url) }, + { abortSignal: upstream.signal, label: safeHostLabel(builtInitialRequest.url) }, ); } } catch (err) { @@ -2475,7 +2483,7 @@ async function handleResponsesInner( const msg = describeUpstreamConnectFailure(err, connectMs); return formatErrorResponse(502, "upstream_error", msg); } finally { - initialRequest.releaseBodyObservation?.(); + builtInitialRequest.releaseBodyObservation?.(); } // Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401 @@ -2581,7 +2589,7 @@ async function handleResponsesInner( route.providerName === "github-copilot" ? getOAuthCredentialApiBaseUrl(route.providerName) : undefined, ); route.provider = refreshedProvider; - transportToken += 1; + invalidateSameTargetRequest(); activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire), config.cacheRetention, @@ -2648,7 +2656,7 @@ async function handleResponsesInner( // until runtime cleanup (one per rotated key under a rate-limit storm). try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } route.provider = rotated; - transportToken += 1; + invalidateSameTargetRequest(); activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire), config.cacheRetention, @@ -2679,7 +2687,7 @@ async function handleResponsesInner( anthropicPoolAccountId = nextAccountId; anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: accessToken }; - transportToken += 1; + invalidateSameTargetRequest(); promoteAnthropicActiveAccount(nextAccountId); logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config); activeAdapter = resolveAdapter( @@ -2704,7 +2712,7 @@ async function handleResponsesInner( })) { imageRetryAttempted = true; imageTierBias = 1; - transportToken += 1; + invalidateSameTargetRequest(); try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } const result = await rebuildAndRefetch("image-413"); if ("failed" in result) return result.failed; @@ -2788,14 +2796,18 @@ async function handleResponsesInner( sameTargetParsed = nextParsed; sameTargetToken = transportToken; } - const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number" - ? continuationRequest.usageLog.inputTokens + // Both branches assign the request (the build catch rethrows), so capture it in a + // const for the fetch callback and finally below — a `let` read inside a nested + // function keeps its undefined half, which would break the byte-identical replay. + const builtContinuationRequest = continuationRequest; + const continuationEstimate = typeof builtContinuationRequest.usageLog?.inputTokens === "number" + ? builtContinuationRequest.usageLog.inputTokens : undefined; if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; try { if (activeAdapter.fetchResponse) { noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replay ? "rate-limit-429" : undefined); - return await activeAdapter.fetchResponse(continuationRequest, { + return await activeAdapter.fetchResponse(builtContinuationRequest, { abortSignal: upstream.signal, timeoutMs: connectMs, stream: nextParsed.stream, @@ -2805,11 +2817,11 @@ async function handleResponsesInner( recovery => { noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery ?? (replay ? "rate-limit-429" : undefined)); return fetchWithHeaderTimeout( - continuationRequest.url, + builtContinuationRequest.url, applyUpstreamRecoveryInit({ - method: continuationRequest.method, - headers: continuationRequest.headers, - body: continuationRequest.body, + method: builtContinuationRequest.method, + headers: builtContinuationRequest.headers, + body: builtContinuationRequest.body, }, recovery), upstream.signal, connectMs, @@ -2817,10 +2829,10 @@ async function handleResponsesInner( providerFetch(route.provider), ); }, - { abortSignal: upstream.signal, label: safeHostLabel(continuationRequest.url) }, + { abortSignal: upstream.signal, label: safeHostLabel(builtContinuationRequest.url) }, ); } finally { - continuationRequest.releaseBodyObservation?.(); + builtContinuationRequest.releaseBodyObservation?.(); } }; while (true) { @@ -2894,7 +2906,7 @@ async function handleResponsesInner( if (rotated) { try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } route.provider = rotated; - transportToken += 1; + invalidateSameTargetRequest(); activeAdapter = resolveAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire), config.cacheRetention, @@ -2921,7 +2933,7 @@ async function handleResponsesInner( anthropicPoolAccountId = nextAccountId; anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: accessToken }; - transportToken += 1; + invalidateSameTargetRequest(); promoteAnthropicActiveAccount(nextAccountId); logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config); activeAdapter = resolveAdapter( @@ -2942,7 +2954,7 @@ async function handleResponsesInner( alreadyAttempted: imageTierBias > 0, })) { imageTierBias = 1; - transportToken += 1; + invalidateSameTargetRequest(); try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } continue; } diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index ea8103b12..ccac2cb2f 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -166,7 +166,7 @@ test("an invalid retryOn429 master switch discards the policy instead of enablin expect(live.providers.test.retryOn429).toBeUndefined(); }); -test("a retryOn429 policy degraded to an empty object still resolves as enabled (presence = opt-in)", () => { +test("a retryOn429 policy with every field invalid is dropped instead of enabling retries", () => { writeDiskConfig({ providers: { test: { @@ -174,15 +174,34 @@ test("a retryOn429 policy degraded to an empty object still resolves as enabled baseUrl: "http://127.0.0.1:1/v1", apiKey: "k", allowPrivateNetwork: true, - // Every field invalid: the sanitizer drops them all and writes back {}. + // Every supplied field invalid: the sanitizer must NOT write back {} — presence + // would opt IN to retries with defaults, the opposite of a disable-oriented + // hand-edit like `attempts: 0`. retryOn429: { attempts: 0 }, }, }, }); const live = loadConfig(); + expect(live.providers.test.retryOn429).toBeUndefined(); + expect(rateLimitRetryPolicyFor(live.providers.test)).toBeNull(); +}); + +test("an intentionally empty retryOn429 policy still resolves as enabled (presence = opt-in)", () => { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: {}, + }, + }, + }); + const live = loadConfig(); expect(live.providers.test.retryOn429).toEqual({}); - // Object presence is the opt-in contract: an emptied object resolves to the enabled defaults, - // exactly like an explicit `retryOn429: {}` in a hand-written config. + // Object presence is the opt-in contract: an explicit `retryOn429: {}` resolves to the + // enabled defaults, exactly like the documented hand-written config. expect(rateLimitRetryPolicyFor(live.providers.test)).toEqual({ enabled: true, attempts: 3, diff --git a/tests/management-provider-validation.test.ts b/tests/management-provider-validation.test.ts index 77ed48206..cf8195b0d 100644 --- a/tests/management-provider-validation.test.ts +++ b/tests/management-provider-validation.test.ts @@ -226,6 +226,14 @@ describe("provider management validation", () => { expect(secretError).toContain("retryOn429 has unrecognized field"); expect(secretError).not.toContain("sk-super-secret-9876"); expect(secretError).toContain("[REDACTED]"); + // A secret-shaped PROVIDER name must not be echoed by the retryOn429 error path either. + const secretNameError = providerManagementConfigError("sk-super-secret-9876", { + ...base, + retryOn429: { attempts: 0 }, + })!; + expect(secretNameError).toContain("retryOn429.attempts is invalid"); + expect(secretNameError).not.toContain("sk-super-secret-9876"); + expect(secretNameError).toContain("[REDACTED]"); }); test("provider discovery status is additive and omitted before an attempt", async () => { From 7a07a2fe8e1a53c9767968cae20470ffea574624 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 20:50:47 +0800 Subject: [PATCH 22/29] docs: attach JSDoc to the new diff-touched declarations Restore CodeRabbit docstring coverage above the 80% threshold for the invalidateSameTargetRequest helper, the RECOVERY_KIND_KEYS map, and recoveryKindKey. --- gui/src/pages/Logs.tsx | 7 +++++++ src/server/responses/core.ts | 8 +++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 17584d7a5..5e4c758bf 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -288,6 +288,10 @@ const ESTIMATE_REASON_KEYS = { expected_price_overlay: "logs.detail.estimate.expected_price_overlay", } as const satisfies Record; +/** + * i18n keys for every {@link AttemptRecoveryKind}, so the logs detail dialog renders a + * localized label instead of the raw wire value (e.g. `rate-limit-429`). + */ const RECOVERY_KIND_KEYS = { "transient-5xx": "logs.detail.attempt.recovery.transient5xx", "connection-reset": "logs.detail.attempt.recovery.connectionReset", @@ -306,6 +310,9 @@ function estimateReasonKey(reason: CostEstimateReason) { return ESTIMATE_REASON_KEYS[reason]; } +/** + * Map one attempt recovery kind to its i18n key for the logs detail dialog. + */ function recoveryKindKey(kind: AttemptRecoveryKind) { return RECOVERY_KIND_KEYS[kind]; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index d296e64b6..d2b188e61 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2450,9 +2450,11 @@ async function handleResponsesInner( let sameTargetParsed: OcxParsedRequest | undefined = parsed; let sameTargetToken = 0; let transportToken = 0; - // Invalidate the same-target request cache. Every credential/adapter/parsed mutation MUST - // go through here: the cache keys on `parsed` REFERENCE identity, so an in-place mutation - // is invisible to it and a missed bump would replay a request built with a stale key. + /** + * Invalidate the same-target request cache. Every credential/adapter/parsed mutation MUST + * go through here: the cache keys on `parsed` REFERENCE identity, so an in-place mutation + * is invisible to it and a missed bump would replay a request built with a stale key. + */ const invalidateSameTargetRequest = (): void => { transportToken += 1; }; let upstreamResponse: Response; try { From 2bd814db31c3e7e940cbcf2eca1fff3d8f4263dd Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 21:05:56 +0800 Subject: [PATCH 23/29] docs: cover the remaining 13 diff-touched declarations with JSDoc CodeRabbit docstring-coverage round: add JSDoc to the six locale catalogs, both AttemptRecoveryKind types, providerConfigSchema, ImageBridgeDeps, WebSearchLoopDeps, DEFAULT_RATE_LIMIT_RETRY, and providerManagementConfigError, bringing assessed coverage to 37/37. --- gui/src/i18n/de.ts | 3 +++ gui/src/i18n/en.ts | 5 +++++ gui/src/i18n/ja.ts | 3 +++ gui/src/i18n/ko.ts | 3 +++ gui/src/i18n/ru.ts | 3 +++ gui/src/i18n/zh.ts | 3 +++ gui/src/pages/Logs.tsx | 4 ++++ src/config.ts | 4 ++++ src/images/loop.ts | 4 ++++ src/providers/key-failover.ts | 4 ++++ src/server/auth-cors.ts | 5 +++++ src/usage/log.ts | 4 ++++ src/web-search/loop.ts | 4 ++++ 13 files changed, 49 insertions(+) diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 4c2998468..687f9de90 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1,6 +1,9 @@ // German — generated from en.ts. Must match TKey set (compile-checked). import type { TKey } from "./en"; +/** + * German i18n catalog, generated from en.ts. Must match the `TKey` set (compile-checked). + */ export const de: Record = { "nav.dashboard": "Übersicht", "nav.startup": "Startsicherheit", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 6788c496e..61d0c93e0 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -1,5 +1,10 @@ // English — source of truth. Its keys define the TKey type; ko/zh/ja must match (compile-checked). // Strings with {cmd} render a chip via ; {var} are plain interpolations. +/** + * English i18n catalog — source of truth. Its keys define the compile-checked `TKey` set; + * other locales must match (compile-checked). `{cmd}` renders a chip via ; + * `{var}` are plain interpolations. + */ export const en = { // sidebar / nav / common "nav.dashboard": "Dashboard", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index fe680ac64..9ce1cac58 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1,5 +1,8 @@ import type { TKey } from "./en"; +/** + * Japanese i18n catalog; must match the `TKey` set (compile-checked). + */ export const ja: Record = { // sidebar / nav / common "nav.dashboard": "ダッシュボード", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index fe92d8be2..45e5e2bc6 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1,5 +1,8 @@ import type { TKey } from "./en"; +/** + * Korean i18n catalog; must match the `TKey` set (compile-checked). + */ export const ko: Record = { // sidebar / nav / common "nav.dashboard": "대시보드", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 8d296a592..98c5e2dfd 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1,5 +1,8 @@ import type { TKey } from "./en"; +/** + * Russian i18n catalog; must match the `TKey` set (compile-checked). + */ export const ru: Record = { // sidebar / nav / common "nav.dashboard": "Дашборд", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 0d8f899b0..9e36305ed 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1,5 +1,8 @@ import type { TKey } from "./en"; +/** + * Chinese i18n catalog; must match the `TKey` set (compile-checked). + */ export const zh: Record = { // sidebar / nav / common "nav.dashboard": "仪表盘", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 5e4c758bf..301077c42 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -76,6 +76,10 @@ interface LogDisplayMetrics { cost: CostResult; } +/** + * Recovery kinds recorded on a log attempt; rendered as localized labels in the logs + * detail dialog instead of raw wire values. + */ type AttemptRecoveryKind = | "transient-5xx" | "connection-reset" diff --git a/src/config.ts b/src/config.ts index 81e4beee9..d0d313a18 100644 --- a/src/config.ts +++ b/src/config.ts @@ -568,6 +568,10 @@ const retryOn429PolicySchema = z.object({ respectRetryAfter: z.boolean().optional(), }).strict(); +/** + * Zod schema for one provider entry: known fields are validated strictly while unknown + * fields pass through (preserved for runtime extensions). + */ const providerConfigSchema = z.object({ adapter: z.string().min(1), baseUrl: z.string().min(1), diff --git a/src/images/loop.ts b/src/images/loop.ts index 6f00a904a..0d84b207c 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -204,6 +204,10 @@ class LoopError extends Error { } } +/** + * Dependencies for one image-bridge iteration: parsed request, active adapter, incoming + * metadata, and the optional image/video bridge plans. + */ export interface ImageBridgeDeps { parsed: OcxParsedRequest; adapter: ProviderAdapter; diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index 2f33cb49e..7a2d2d330 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -22,6 +22,10 @@ interface KeyCooldown { const DEFAULT_COOLDOWN_MS = 60_000; const MAX_COOLDOWN_MS = 10 * 60_000; // cap at 10 min for api-key rotation +/** + * Default same-target 429 retry policy used when a provider opts in via a bare + * `retryOn429: {}` (presence = opt-in with these defaults). + */ const DEFAULT_RATE_LIMIT_RETRY = { enabled: true, attempts: 3, diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index 3fe1a6c6a..2acea2848 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -391,6 +391,11 @@ function sameCanonicalProviderSeed(actual: Record, expected: Oc return actualKeys.every(key => JSON.stringify(actual[key]) === JSON.stringify((expected as unknown as Record)[key])); } +/** + * Validate a provider object arriving at the management write boundary. Returns an error + * string, or null when the provider may be persisted. Caller-controlled names/fields are + * redacted and JSON-escaped so secrets never reach the response. + */ export function providerManagementConfigError(name: unknown, provider: unknown): string | null { if (typeof name !== "string" || !provider || typeof provider !== "object" || Array.isArray(provider)) { return "provider must be a plain object"; diff --git a/src/usage/log.ts b/src/usage/log.ts index 3cfab3a67..9a66422b8 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -7,6 +7,10 @@ import type { OcxUsage } from "../types"; export type UsageStatus = "reported" | "unreported" | "unsupported" | "estimated"; +/** + * Recovery kinds recorded per attempt in the usage log; the GUI renders localized labels + * for these wire values. + */ export type AttemptRecoveryKind = | "transient-5xx" | "connection-reset" diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 5bd96fab6..7c918c6df 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -208,6 +208,10 @@ class LoopError extends Error { } } +/** + * Dependencies for one web-search loop iteration: parsed request, active adapter, + * incoming metadata, and the configured search executor. + */ export interface WebSearchLoopDeps { parsed: OcxParsedRequest; adapter: ProviderAdapter; From d2db429516401efb393c45c1f4674e553351d66b Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 21:18:15 +0800 Subject: [PATCH 24/29] fix(proxy): address formal-review findings across core, config, GUI, and docs - core.ts: hoist imageTierBias to shared scope so a 413-driven tier reduction survives into the terminal-guard continuation (avoid redundant 413 round trip); consolidate the replay-derived recovery label into one replayKind value. - gui Logs.tsx + locales: localized fallback for unknown recovery kinds (logs.detail.attempt.recovery.unknown) in every locale; use the established Korean request-throttling wording for the 429 recovery labels. - config.ts: sanitize retryOn429 in configDiagnosticsFromRaw before schema validation so an invalid hand-edit cannot push the diagnostics path to a default fallback that the config command would persist over user providers; regression test added. - docs: restore leading pipes on all malformed ja providers.md table separator rows (MD055); sync the Copilot mixed-wire paragraph into ja/ko/ru/zh-cn guides before the Cursor sections; document the runTurn exception for bridge retries in the 429 design devlog. --- .../010_design.md | 3 +++ .../src/content/docs/ja/guides/providers.md | 19 ++++++++++++----- .../src/content/docs/ko/guides/providers.md | 9 ++++++++ .../src/content/docs/ru/guides/providers.md | 9 ++++++++ .../content/docs/zh-cn/guides/providers.md | 8 +++++++ gui/src/i18n/de.ts | 1 + gui/src/i18n/en.ts | 1 + gui/src/i18n/ja.ts | 1 + gui/src/i18n/ko.ts | 7 ++++--- gui/src/i18n/ru.ts | 1 + gui/src/i18n/zh.ts | 1 + gui/src/pages/Logs.tsx | 4 +++- src/config.ts | 4 ++++ src/server/responses/core.ts | 11 ++++++---- tests/config-user-edits.test.ts | 21 +++++++++++++++++++ 15 files changed, 87 insertions(+), 13 deletions(-) diff --git a/devlog/_plan/260802_429_same_target_retry/010_design.md b/devlog/_plan/260802_429_same_target_retry/010_design.md index 6067587df..6b390c2b9 100644 --- a/devlog/_plan/260802_429_same_target_retry/010_design.md +++ b/devlog/_plan/260802_429_same_target_retry/010_design.md @@ -51,6 +51,9 @@ existing multi-key failover runs. Default off → zero behavior change for exist - image/video bridge and web-search sidecar loops (`src/images/loop.ts`, `src/web-search/loop.ts`) — before their `on429` key rotation; - Anthropic terminal-guard continuations — before key/account failover. + Bridge retries apply to HTTP adapters only: a custom transport that enters the + `adapter.runTurn` branch (`src/images/loop.ts`) returns before the HTTP 429 retry loop and + therefore does not receive the wait-and-replay policy. Every surface releases (and awaits the cancellation of) the unread 429 body BEFORE the backoff, records the `rate-limit-429` recovery kind on replay sends, and (bridges) clears the old response-header deadline before the wait and starts a fresh one afterward, re-checking diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index f6b311281..be6f78ea8 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -9,7 +9,7 @@ description: opencodex が LLM プロバイダーを認証し通信するすべ ## OpenAI アカウントモード | プロバイダー ID | 用途 | 認証情報/アカウットルール | - --- | --- | --- | +| --- | --- | --- | | `openai` | Codex ログイン | Pool(デフォルト)はメイン + 追加アカウントを選び、Direct は現在の caller/メインログインのみを使います。 | | `openai-apikey` | OpenAI API | 設定された API キー/キープールのみを使い、Codex アカウントは読みません。 | @@ -30,7 +30,7 @@ max input 922,000 で `*-pro` virtual ID は公開状態を維持し、wire で ローカルプリセットを別に分類します。ローカルプリセットでは通常 `authMode` と `apiKey` を両方使いません。 | `authMode` | 認証方式 | 用途 | - --- | --- | --- | +| --- | --- | --- | | `key` | API キーを送信します(`Authorization: Bearer …`、またはアダプターにより `x-api-key` / `api-key`)。キーはリテラルまたは `${ENV_VAR}` 参照です。 | 大半のプロバイダー。 | | `forward` | **受け取った Codex 認証ヘッダーを**プロバイダーにそのまま中継します — キーを保存しません。ChatGPT ログインのパススルーです。 | OpenAI(`openai-responses` アダプター)。 | | `oauth` | 保存された OAuth アクセストークンを読み込み bearer キーとして使い、期限切れ前に自動更新します。 | xAI、Anthropic、Kimi、Kiro、Google Antigravity、Cursor。 | @@ -143,7 +143,7 @@ Cline IDE/CLI のみで API からは使えません。`minimax/minimax-m2.5` 無料試用モデルとして文書化されています。 | プロバイダー | ベース URL | - --- | --- | +| --- | --- | | **OpenAI (API キー)** | `https://api.openai.com/v1` | | **Anthropic (API キー)** | `https://api.anthropic.com` | | **OpenRouter** | `https://openrouter.ai/api/v1` | @@ -243,7 +243,7 @@ OAuth、API キープールを確認・切り替えできます。完全なコ Sol/Terra/Luna をフォールバックリストに入れています。 | Codex 経路 | 事前登録されたモデル ID | Codex に表示されるコンテキスト | - --- | --- | --- | +| --- | --- | --- | | Codex ログイン(Pool または Direct) | `gpt-5.6-*` | 372,000 | | OpenAI (API キー) | `openai-apikey/gpt-5.6-*` と `*-pro` | 1,050,000 (max input 922,000) | | OpenRouter | `openrouter/openai/gpt-5.6-sol`、`openrouter/openai/gpt-5.6-terra`、`openrouter/openai/gpt-5.6-luna` | 1,050,000 | @@ -265,6 +265,15 @@ Amazon Bedrock ネイティブ API のような、これらの実装のいずれ **サブスクリプショントークン**(通常の API キーではない)で認証します。**Cloudflare AI Gateway** は URL にアカウント + ゲートウェイ ID を埋める必要があります。 +Copilot は混在 wire カタログを提供します。GPT-5 系モデル(`gpt-5.3-codex`、`gpt-5.4`、 +`gpt-5.4-mini`、`gpt-5.5`、`gpt-5.6-luna`、`gpt-5.6-sol`、`gpt-5.6-terra`)はエージェント +通信の `/chat/completions` を拒否するため、opencodex はこれらのモデルを組み込みデフォルトで +Responses API 経由にルーティングし、他の Copilot モデルはすべて chat completions のままです。 +優先順位は次のとおりです: ハード wire ピン → 明示的な +[`modelAdapters`](/reference/configuration/providers/) エントリ → レジストリのデフォルト → +プロバイダー全体の adapter。組み込みデフォルトのないモデル(例: `gpt-5.4-nano`)を Responses +に移すには、`"modelAdapters": { "gpt-5.4-nano": "openai-responses" }` を設定してください。 + Cursor は別の実験的アダプターとして追跡します。`adapter: "cursor"` は `ocx init` とダッシュボード Add Provider ピッカーに実験的 local config 項目として表示され、Cursor の静的フォールバックモデルカタログ メタデータを保存します。Cursor アクセストークンを設定すると opencodex は Cursor ライブ HTTP/2 トランスポートを @@ -298,7 +307,7 @@ Ollama の `:size` タグに寛容なので `gpt-oss` は `gpt-oss:120b` と `gp opencodex をローカルの OpenAI 互換サーバーに向けてください — 通常は空キーで使います: | プロバイダー | ベース URL | - --- | --- | +| --- | --- | | Ollama (local) | `http://localhost:11434/v1` | | vLLM | `http://localhost:8000/v1` | | LM Studio | `http://localhost:1234/v1` | diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 79dc95ff5..8d75d55bc 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -266,6 +266,15 @@ Amazon Bedrock 네이티브 API처럼 이 구현 중 어느 것과도 맞지 않 **구독 토큰**(일반 API 키가 아님)으로 인증합니다. **Cloudflare AI Gateway**는 URL에 계정 + 게이트웨이 id를 채워야 합니다. +Copilot은 혼합 wire 카탈로그를 제공합니다. GPT-5 계열 모델(`gpt-5.3-codex`, `gpt-5.4`, +`gpt-5.4-mini`, `gpt-5.5`, `gpt-5.6-luna`, `gpt-5.6-sol`, `gpt-5.6-terra`)은 에이전트 +트래픽에 대해 `/chat/completions`를 거부하므로 opencodex는 이 모델들을 내장 기본값으로 +Responses API를 통해 라우팅하고, 다른 Copilot 모델은 모두 chat completions를 유지합니다. +우선순위는 하드 wire 핀 → 명시적 [`modelAdapters`](/reference/configuration/providers/) +항목 → 레지스트리 기본값 → 프로바이더 전체 adapter 순입니다. 내장 기본값이 없는 모델(예: +`gpt-5.4-nano`)을 Responses로 전환하려면 `"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`를 +설정하세요. + Cursor는 별도의 실험적 어댑터로 추적합니다. `adapter: "cursor"`는 `ocx init`과 dashboard Add Provider picker에 실험적 local config 항목으로 표시되며, Cursor의 static fallback model catalog metadata를 저장합니다. Cursor access token이 설정되면 opencodex는 Cursor live HTTP/2 transport를 diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 612a733d8..893fd201a 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -278,6 +278,15 @@ Assist), `azure` / `azure-openai`, `kiro` и `cursor`. Проприетарны **GitLab Duo** остаётся шлюзом с ключом/токеном подписки на своей OpenAI-совместимой конечной точке. **Cloudflare AI Gateway** требует подставить в URL id аккаунта и шлюза. +Copilot предоставляет каталог со смешанными проводами: его семейство GPT-5 (`gpt-5.3-codex`, +`gpt-5.4`, `gpt-5.4-mini`, `gpt-5.5`, `gpt-5.6-luna`, `gpt-5.6-sol`, `gpt-5.6-terra`) +отклоняет `/chat/completions` для агентного трафика, поэтому opencodex по умолчанию +маршрутизирует эти модели через Responses API, а все остальные модели Copilot остаются на +chat completions. Приоритет: жёсткий wire-пин → явная запись +[`modelAdapters`](/reference/configuration/providers/) → дефолт реестра → adapter всего +провайдера. Чтобы перевести модель без встроенного дефолта (например, `gpt-5.4-nano`) на +Responses, задайте `"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`. + Cursor отслеживается отдельно как экспериментальный адаптер. `adapter: "cursor"` появляется в `ocx init` и в селекторе Add Provider дашборда как экспериментальная запись локальной конфигурации с метаданными статического резервного каталога моделей Cursor. Когда настроен токен доступа Cursor, diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index f5fedae06..2335c2b9c 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -246,6 +246,14 @@ GPT-5.6 Sol/Terra/Luna 会预置在提供商的回退列表中,因此即使实 使用 Bearer **订阅令牌**(而非普通 API 密钥)进行认证。 **Cloudflare AI Gateway** 需要将 account 和 gateway id 填入 URL。 +Copilot 提供混合 wire 目录:其 GPT-5 系列模型(`gpt-5.3-codex`、`gpt-5.4`、 +`gpt-5.4-mini`、`gpt-5.5`、`gpt-5.6-luna`、`gpt-5.6-sol`、`gpt-5.6-terra`)会拒绝面向 +agent 流量的 `/chat/completions`,因此 opencodex 默认将这些模型路由到 Responses API,而其他 +Copilot 模型仍走 chat completions。优先级为:硬 wire 固定 → 显式 +[`modelAdapters`](/reference/configuration/providers/) 条目 → 注册表默认值 → 提供商级 +adapter。若要将没有内置默认值的模型(例如 `gpt-5.4-nano`)接入 Responses,请设置 +`"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`。 + Cursor 作为单独的实验性 adapter 进行跟踪。`adapter: "cursor"` 会作为实验性本地配置出现在 `ocx init` 和 dashboard Add Provider picker 中,并保存 Cursor 的静态回退模型目录 metadata。配置 Cursor access token 后,opencodex 会使用 Cursor live HTTP/2 transport。内置回退列表包含上下文为 diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 687f9de90..10140a0c6 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -559,6 +559,7 @@ export const de: Record = { "logs.detail.attempt.recovery.rateLimit429": "Ratenbegrenzt (429)", "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth ratenbegrenzt (429)", "logs.detail.attempt.recovery.image413": "Bildnutzlast zu groß (413)", + "logs.detail.attempt.recovery.unknown": "Unbekannter Wiederherstellungsgrund", "logs.detail.reason.usage_missing": "Nutzung wurde nicht gemeldet.", "logs.detail.reason.usage_unsupported": "Dieser Anbieter meldet keine Nutzung.", "logs.detail.reason.output_missing": "Es wurden keine positiven Ausgabe-Tokens gemeldet.", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 61d0c93e0..9e2b519bd 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -586,6 +586,7 @@ export const en = { "logs.detail.attempt.recovery.rateLimit429": "Rate-limited (429)", "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth rate-limited (429)", "logs.detail.attempt.recovery.image413": "Image payload too large (413)", + "logs.detail.attempt.recovery.unknown": "Unknown recovery reason", "logs.detail.reason.usage_missing": "Usage was not reported.", "logs.detail.reason.usage_unsupported": "This provider does not report usage.", "logs.detail.reason.output_missing": "No positive output token count was reported.", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 9ce1cac58..6b8d52934 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -544,6 +544,7 @@ export const ja: Record = { "logs.detail.attempt.recovery.rateLimit429": "レート制限 (429)", "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth レート制限 (429)", "logs.detail.attempt.recovery.image413": "画像ペイロードが大きすぎます (413)", + "logs.detail.attempt.recovery.unknown": "不明なリカバリ理由", "logs.detail.reason.usage_missing": "使用量が報告されませんでした。", "logs.detail.reason.usage_unsupported": "このプロバイダーは使用量を報告しません。", "logs.detail.reason.output_missing": "正の出力トークン数が報告されませんでした。", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 45e5e2bc6..654ef4f4a 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -574,10 +574,11 @@ export const ko: Record = { "logs.detail.attempt.recovery.transient5xx": "일시적 5xx 오류", "logs.detail.attempt.recovery.connectionReset": "연결이 재설정됨", "logs.detail.attempt.recovery.oauth401": "OAuth 재인증", - "logs.detail.attempt.recovery.key429": "키 요금 제한 (429)", - "logs.detail.attempt.recovery.rateLimit429": "요금 제한 (429)", - "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth 요금 제한 (429)", + "logs.detail.attempt.recovery.key429": "키 요청 한도 초과 (429)", + "logs.detail.attempt.recovery.rateLimit429": "요청 한도 초과 (429)", + "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth 요청 한도 초과 (429)", "logs.detail.attempt.recovery.image413": "이미지 페이로드가 너무 큼 (413)", + "logs.detail.attempt.recovery.unknown": "알 수 없는 복구 사유", "logs.detail.reason.usage_missing": "usage가 보고되지 않았습니다.", "logs.detail.reason.usage_unsupported": "이 프로바이더는 usage 보고를 지원하지 않습니다.", "logs.detail.reason.output_missing": "양수 출력 토큰 수가 보고되지 않았습니다.", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 98c5e2dfd..56c52ecd0 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -576,6 +576,7 @@ export const ru: Record = { "logs.detail.attempt.recovery.rateLimit429": "Ограничение частоты запросов (429)", "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth ограничен (429)", "logs.detail.attempt.recovery.image413": "Слишком большой размер изображения (413)", + "logs.detail.attempt.recovery.unknown": "Неизвестная причина восстановления", "logs.detail.reason.usage_missing": "Данные об использовании не были сообщены.", "logs.detail.reason.usage_unsupported": "Этот провайдер не сообщает данные об использовании.", "logs.detail.reason.output_missing": "Положительное число выходных токенов не было сообщено.", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 9e36305ed..e9860f45b 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -571,6 +571,7 @@ export const zh: Record = { "logs.detail.attempt.recovery.rateLimit429": "被限流 (429)", "logs.detail.attempt.recovery.anthropicOauth429": "Anthropic OAuth 被限流 (429)", "logs.detail.attempt.recovery.image413": "图片载荷过大 (413)", + "logs.detail.attempt.recovery.unknown": "未知的恢复原因", "logs.detail.reason.usage_missing": "未上报 usage。", "logs.detail.reason.usage_unsupported": "该提供方不支持上报 usage。", "logs.detail.reason.output_missing": "未上报正数输出 token。", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 301077c42..ebe177532 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -318,7 +318,9 @@ function estimateReasonKey(reason: CostEstimateReason) { * Map one attempt recovery kind to its i18n key for the logs detail dialog. */ function recoveryKindKey(kind: AttemptRecoveryKind) { - return RECOVERY_KIND_KEYS[kind]; + // A stale/malformed cached row can carry a kind outside the known set; fall back to a + // localized label instead of handing `t()` an undefined key. + return RECOVERY_KIND_KEYS[kind] ?? "logs.detail.attempt.recovery.unknown"; } function verificationKey(status: MatchedPriceInfo["status"]): "logs.detail.verification.verified" | "logs.detail.verification.derived" { diff --git a/src/config.ts b/src/config.ts index d0d313a18..2a11d0d36 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1786,6 +1786,10 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx function configDiagnosticsFromRaw(raw: string): ConfigDiagnostics { try { const parsed = JSON.parse(raw.replace(/^\uFEFF/, "")); + // Same degradation as loadConfig: a hand-edited invalid retryOn429 must not trip the + // schema and send the caller a default-config fallback (the config command could then + // persist that fallback over the user's providers/keys). + sanitizeRetryOn429ForLoad(parsed); const result = configSchema.safeParse(parsed); if (result.success) { return validFileConfigDiagnostics(normalizeApiKeyIds(result.data as OcxConfig), parsed); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index d2b188e61..318cb647b 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2494,13 +2494,15 @@ async function handleResponsesInner( // `attempts` same-key replays in total (bounded per request). const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); let rateLimitRetries = 0; + // Shared with the terminal-guard continuation below: an image-tier reduction that let the + // main request clear a 413 must not be forgotten on the very next continuation build. + let imageTierBias = 0; if (!upstreamResponse.ok) { // Recovery loop: multi-key 429 failover + at most ONE anthropic 413 tightened retry // (devlog/260714_image_normalization_pipeline/030). One mutable activeAdapter serves // both paths so a 429→413 sequence never rebuilds against a stale pre-rotation // adapter, and imageTierBias — once armed — rides EVERY subsequent rebuild so a // 413→429 rotation cannot silently undo the tightening. - let imageTierBias = 0; let imageRetryAttempted = false; let oauth401ReplayAttempted = false; /** @@ -2767,7 +2769,6 @@ async function handleResponsesInner( * never sees a second hidden HTTP response or an unbounded retry loop. */ const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { - let imageTierBias = 0; let response: Response | undefined; /** * Build and fetch one terminal-guard continuation. `replay` marks a same-target 429 @@ -2806,9 +2807,11 @@ async function handleResponsesInner( ? builtContinuationRequest.usageLog.inputTokens : undefined; if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; + // One label rule for continuation replays, shared by both send paths below. + const replayKind: AttemptRecoveryKind | undefined = replay ? "rate-limit-429" : undefined; try { if (activeAdapter.fetchResponse) { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replay ? "rate-limit-429" : undefined); + noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replayKind); return await activeAdapter.fetchResponse(builtContinuationRequest, { abortSignal: upstream.signal, timeoutMs: connectMs, @@ -2817,7 +2820,7 @@ async function handleResponsesInner( } return await fetchWithResetRetry( recovery => { - noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery ?? (replay ? "rate-limit-429" : undefined)); + noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery ?? replayKind); return fetchWithHeaderTimeout( builtContinuationRequest.url, applyUpstreamRecoveryInit({ diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index ccac2cb2f..207833622 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -6,6 +6,7 @@ import { armClaudeCodeBaseline, getConfigPath, loadConfig, + readConfigDiagnostics, reconcileLiveConfigFromDisk, saveConfig, saveConfigPreservingClaudeCode, @@ -211,6 +212,26 @@ test("an intentionally empty retryOn429 policy still resolves as enabled (presen }); }); +test("config diagnostics sanitize invalid retryOn429 before schema validation", () => { + writeDiskConfig({ + providers: { + test: { + adapter: "openai-chat", + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "k", + allowPrivateNetwork: true, + retryOn429: { attempts: 0 }, + }, + }, + }); + const diagnostics = readConfigDiagnostics(); + // Without sanitization the schema rejects the config and the diagnostics path returns a + // default fallback, which the config command could persist over the user's providers. + expect(diagnostics.source).not.toBe("fallback"); + expect(diagnostics.config.providers.test).toBeDefined(); + expect(diagnostics.config.providers.test.retryOn429).toBeUndefined(); +}); + test("invalid retryOn429 values never log the raw value", () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); try { From 6c7ea9f77e5fd66c24af0f1ddfccb8096b7d67df Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 21:39:14 +0800 Subject: [PATCH 25/29] docs(locale): keep the Copilot modelAdapters links inside each locale tree The translated Copilot paragraphs referenced the English /reference/... path; use the locale-prefixed /ja|ko|ru|zh-cn/reference/configuration/providers/ links matching the rest of each guide. --- docs-site/src/content/docs/ja/guides/providers.md | 2 +- docs-site/src/content/docs/ko/guides/providers.md | 2 +- docs-site/src/content/docs/ru/guides/providers.md | 2 +- docs-site/src/content/docs/zh-cn/guides/providers.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index be6f78ea8..0fa811e4a 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -270,7 +270,7 @@ Copilot は混在 wire カタログを提供します。GPT-5 系モデル(`gp 通信の `/chat/completions` を拒否するため、opencodex はこれらのモデルを組み込みデフォルトで Responses API 経由にルーティングし、他の Copilot モデルはすべて chat completions のままです。 優先順位は次のとおりです: ハード wire ピン → 明示的な -[`modelAdapters`](/reference/configuration/providers/) エントリ → レジストリのデフォルト → +[`modelAdapters`](/ja/reference/configuration/providers/) エントリ → レジストリのデフォルト → プロバイダー全体の adapter。組み込みデフォルトのないモデル(例: `gpt-5.4-nano`)を Responses に移すには、`"modelAdapters": { "gpt-5.4-nano": "openai-responses" }` を設定してください。 diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 8d75d55bc..032766ab5 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -270,7 +270,7 @@ Copilot은 혼합 wire 카탈로그를 제공합니다. GPT-5 계열 모델(`gpt `gpt-5.4-mini`, `gpt-5.5`, `gpt-5.6-luna`, `gpt-5.6-sol`, `gpt-5.6-terra`)은 에이전트 트래픽에 대해 `/chat/completions`를 거부하므로 opencodex는 이 모델들을 내장 기본값으로 Responses API를 통해 라우팅하고, 다른 Copilot 모델은 모두 chat completions를 유지합니다. -우선순위는 하드 wire 핀 → 명시적 [`modelAdapters`](/reference/configuration/providers/) +우선순위는 하드 wire 핀 → 명시적 [`modelAdapters`](/ko/reference/configuration/providers/) 항목 → 레지스트리 기본값 → 프로바이더 전체 adapter 순입니다. 내장 기본값이 없는 모델(예: `gpt-5.4-nano`)을 Responses로 전환하려면 `"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`를 설정하세요. diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 893fd201a..1be9218c0 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -283,7 +283,7 @@ Copilot предоставляет каталог со смешанными пр отклоняет `/chat/completions` для агентного трафика, поэтому opencodex по умолчанию маршрутизирует эти модели через Responses API, а все остальные модели Copilot остаются на chat completions. Приоритет: жёсткий wire-пин → явная запись -[`modelAdapters`](/reference/configuration/providers/) → дефолт реестра → adapter всего +[`modelAdapters`](/ru/reference/configuration/providers/) → дефолт реестра → adapter всего провайдера. Чтобы перевести модель без встроенного дефолта (например, `gpt-5.4-nano`) на Responses, задайте `"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`. diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index 2335c2b9c..6ce56c9bc 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -250,7 +250,7 @@ Copilot 提供混合 wire 目录:其 GPT-5 系列模型(`gpt-5.3-codex`、`g `gpt-5.4-mini`、`gpt-5.5`、`gpt-5.6-luna`、`gpt-5.6-sol`、`gpt-5.6-terra`)会拒绝面向 agent 流量的 `/chat/completions`,因此 opencodex 默认将这些模型路由到 Responses API,而其他 Copilot 模型仍走 chat completions。优先级为:硬 wire 固定 → 显式 -[`modelAdapters`](/reference/configuration/providers/) 条目 → 注册表默认值 → 提供商级 +[`modelAdapters`](/zh-cn/reference/configuration/providers/) 条目 → 注册表默认值 → 提供商级 adapter。若要将没有内置默认值的模型(例如 `gpt-5.4-nano`)接入 Responses,请设置 `"modelAdapters": { "gpt-5.4-nano": "openai-responses" }`。 From 4650baef6c323770622ee2d6055c0b8cbb3c6e0d Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Mon, 3 Aug 2026 23:04:23 +0800 Subject: [PATCH 26/29] docs: reattach JSDoc to fetchOnce and document the abort hook The shared-request cache refactor inserted the cachedRequest/cachedAdapter declarations between the fetchOnce JSDoc and the function itself, orphaning the doc block; reattach an adjacent JSDoc to both fetchOnce declarations and add one to the bounded-body onAbort hook so the declarations stay documented. --- src/images/loop.ts | 4 ++++ src/lib/upstream-retry.ts | 4 ++++ src/web-search/loop.ts | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/images/loop.ts b/src/images/loop.ts index 0d84b207c..8bd668907 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -451,6 +451,10 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise => { let request: AdapterRequest; if (cachedRequest !== undefined && cachedAdapter === requestAdapter) { diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 670778af1..06d14c3ef 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -98,6 +98,10 @@ export async function releaseResponseBodyBestEffort( } await new Promise(resolve => { let timer: ReturnType; + /** + * Abort hook: clear the bounded-body release timer and settle the promise so a + * never-settling cancel() can never block the abort-aware backoff. + */ const onAbort = () => { clearTimeout(timer); resolve(); diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 7c918c6df..203e00178 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -350,6 +350,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise => { let request: AdapterRequest; if (cachedRequest !== undefined && cachedAdapter === requestAdapter) { From 6fb4fe265ca18593c6cb9280caf34c3fcb071323 Mon Sep 17 00:00:00 2001 From: HarryZhou <2373256746@qq.com> Date: Tue, 4 Aug 2026 09:44:24 +0800 Subject: [PATCH 27/29] fix(bridge): complete send telemetry; clarify retry docs scope CodeRabbit round on 4650baef: - images/web-search loops: move onAttemptSend into the fetchWithResetRetry callback (retryRecovery ?? recovery) so helper-driven connection-reset sends are recorded; classify rotated-adapter fetches as key-429 like core recovery. - providers docs (en/ja/ko/ru/zh-cn): state explicitly that exhausting attempts only stops same-key replays, with normal failover or final-error handling next. - structure/04: scope retryOn429 to HTTP-capable API-key adapters and exclude custom runTurn transports. - ko/zh-cn providers ref: keep routing links inside each locale tree. --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 4 ++-- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 4 ++-- src/images/loop.ts | 22 ++++++++++++------- src/web-search/loop.ts | 22 ++++++++++++------- structure/04_transports-and-sidecars.md | 4 +++- 8 files changed, 38 insertions(+), 24 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 16edaf869..0285abddb 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -85,7 +85,7 @@ namespace 付き combo alias はその namespace prefix に selector を再利 | `noPenaltyModels?` | `string[]` |存在/周波数ペナルティを拒否するモデル。 | | `parallelToolCalls?` | `boolean` |並列ツール呼び出しを切り替えます。 OpenAI Chat はデフォルトでオンになっています。非チャット アダプターは明示的な `true` でのみアドバタイズします。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` |正確なプレースホルダー ID および欠落している端末 ID に対するダウンストリーム SSE 修復はデフォルトで無効になっています。関数呼び出し ID は決して書き換えられません。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。キー認証の passthrough ワイヤにはキー フェイルオーバーがないため、予算を使い切ると 429 がそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。`attempts` を使い切っても同一キーでの再送が止まるだけで、通常のキー フェイルオーバーまたは最終エラー処理が利用可能なターゲットに応じて続きます — キー認証の passthrough ワイヤにはフェイルオーバーがないため、使い切った 429 はそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` が `auto` または `none` のみを受け入れるモデル。強制的な選択は格下げされます。 | | `preserveReasoningContentModels?` | `string[]` |チャット履歴に以前のアシスタント `reasoning_content` が必要なモデル。 | | `thinkingToggleModels?` | `string[]` |エフォート ラダーではなく `thinking.enabled` を使用してモデルをチャットします。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index fdbba3ce4..38a499c6c 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -33,7 +33,7 @@ provider 및 예약된 `openai` / `combo` 충돌은 대소문자를 구분하지 combo alias는 selector를 namespace prefix로 재사용할 수 없습니다. 설정된 pool id와 다른 selector target도 selector로 재사용할 수 없습니다. raw account id와 email은 비공개로 유지하고 selector를 공개 이름으로 사용하세요. 명시적 선택 동작과 우선순위는 -[라우팅 설정](/reference/configuration/routing/)을 참고하십시오. +[라우팅 설정](/ko/reference/configuration/routing/)을 참고하십시오. ## 예약된 OpenAI 공급자 @@ -85,7 +85,7 @@ target도 selector로 재사용할 수 없습니다. raw account id와 email은 | `noPenaltyModels?` | `string[]` | presence/frequency penalty를 허용하지 않는 모델입니다. | | `parallelToolCalls?` | `boolean` | 병렬 도구 호출을 켜거나 끕니다. OpenAI Chat은 기본으로 켜져 있고, 비-chat 어댑터는 명시적으로 `true`일 때만 이를 노출합니다. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 기본값이 꺼진 downstream SSE 복구입니다. 정확한 자리표시자 id와 누락된 종료 id를 복구합니다. function-call id는 다시 쓰지 않습니다. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. 키 인증 passthrough 와이어에는 키 장애 조치가 없으므로 예산을 모두 쓰면 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. `attempts`를 모두 소진해도 동일 키 재전송만 중단되며, 이후에는 일반 키 장애 조치 또는 최종 오류 처리가 사용 가능한 대상에 따라 진행됩니다 — 키 인증 passthrough 와이어에는 장애 조치가 없으므로 소진된 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. | | `thinkingToggleModels?` | `string[]` | effort 계층 대신 `thinking.enabled`를 쓰는 chat 모델입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 5bd538ca9..2611c09cd 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -92,7 +92,7 @@ differing backup and rewrites known legacy namespaced selected ids to bare ids. | `noPenaltyModels?` | `string[]` | Models that reject presence/frequency penalties. | | `parallelToolCalls?` | `boolean` | Toggle parallel tool calls. OpenAI Chat defaults on; non-chat adapters advertise only on explicit `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Disabled-by-default downstream SSE repair for exact placeholder ids and missing terminal ids. Function-call ids are never rewritten. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries; on the key-auth passthrough wire there is no key failover, so an exhausted budget surfaces the 429. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries; Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is.. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | | `thinkingToggleModels?` | `string[]` | Chat models using `thinking.enabled` rather than an effort ladder. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 3cf76ae88..be60250f6 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -95,7 +95,7 @@ cross-route credential fallback не существует. Строки API GPT- | `noPenaltyModels?` | `string[]` | Модели, отвергающие penalty presence/frequency. | | `parallelToolCalls?` | `boolean` | Переключатель parallel tool call'ов. Для OpenAI Chat по умолчанию включено; не-chat adapter'ы рекламируют это только при явном `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | По умолчанию выключенная downstream SSE-repair для exact placeholder-id и отсутствующих terminal-id. Function-call id никогда не переписываются. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста; на passthrough-канале с ключевой аутентификацией фейловера ключей нет, поэтому при исчерпании бюджета 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста; Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие `thinking.enabled` вместо effort-ladder. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 36bbb67d2..4d672e90e 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -32,7 +32,7 @@ pool account id(不能是内部 `__main__`),或用 `"@main"` 表示 Codex 保留的 `openai` / `combo` 冲突时不区分大小写;带 namespace 的 combo alias 不能把 selector 复用为 其 namespace prefix,已配置的 pool id 和其他 selector target 也不能复用为 selector。raw account id 与 email 应保持私密,selector 才是公开名称。明确选择的行为和优先级见 -[路由配置](/reference/configuration/routing/)。 +[路由配置](/zh-cn/reference/configuration/routing/)。 ## 保留的 OpenAI 提供者 @@ -84,7 +84,7 @@ pool account id(不能是内部 `__main__`),或用 `"@main"` 表示 Codex | `noPenaltyModels?` | `string[]` | 会拒绝 presence/frequency penalty 的模型。 | | `parallelToolCalls?` | `boolean` | 切换并行工具调用。OpenAI Chat 默认开启;非 chat 适配器只有显式 `true` 时才会声明支持。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 默认关闭的下游 SSE 修复,用于精确占位 id 和缺失的终止 id。function-call id 永远不会被重写。 | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应;自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数(总发送次数 = `attempts` + 1),是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算;key 认证的 passthrough 线路上没有 key 故障转移,因此预算耗尽时 429 会原样透出。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应;自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数(总发送次数 = `attempts` + 1),是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算;`attempts` 耗尽只会停止进一步的同 key 重放:随后按可用目标进行正常的 key 故障转移或最终错误处理——key 认证的 passthrough 线路上没有故障转移,因此耗尽的 429 会原样透出。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型;强制选择会被降级。 | | `preserveReasoningContentModels?` | `string[]` | 需要在聊天历史中保留先前 assistant `reasoning_content` 的模型。 | | `thinkingToggleModels?` | `string[]` | 使用 `thinking.enabled` 而不是 effort 阶梯的 chat 模型。 | diff --git a/src/images/loop.ts b/src/images/loop.ts index 8bd668907..5fbaeb038 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -469,18 +469,23 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise { + }); + } else { + response = await fetchWithResetRetry( + (retryRecovery) => { + // Record every helper-driven send (the callback runs for the first attempt and + // each connection-reset replay); preserve the caller's recovery kind + // (rate-limit-429 / key-429) when the retry layer supplies none. + deps.onAttemptSend?.(retryRecovery ?? recovery); const h = new Headers(request.headers); if (!h.has("accept-encoding")) h.set("accept-encoding", "identity"); return fetchImpl(request.url, { @@ -491,7 +496,8 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } adapter = rotated; yield { type: "heartbeat" }; - prepared = await fetchOnce(adapter); + prepared = await fetchOnce(adapter, "key-429"); } // Final headers have arrived. Clear only the deadline timer before ANY body read. diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 203e00178..90a04acf6 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -372,18 +372,23 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { + }); + } else { + response = await fetchWithResetRetry( + (retryRecovery) => { + // Record every helper-driven send (the callback runs for the first attempt and + // each connection-reset replay); preserve the caller's recovery kind + // (rate-limit-429 / key-429) when the retry layer supplies none. + deps.onAttemptSend?.(retryRecovery ?? recovery); const h = new Headers(request.headers); if (!h.has("accept-encoding")) h.set("accept-encoding", "identity"); return fetch(request.url, { @@ -394,7 +399,8 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise Date: Tue, 4 Aug 2026 09:52:16 +0800 Subject: [PATCH 28/29] docs(providers): fix punctuation in the retryOn429 exhaustion sentence --- docs-site/src/content/docs/reference/configuration/providers.md | 2 +- .../src/content/docs/ru/reference/configuration/providers.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 2611c09cd..7e27d048b 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -92,7 +92,7 @@ differing backup and rewrites known legacy namespaced selected ids to bare ids. | `noPenaltyModels?` | `string[]` | Models that reject presence/frequency penalties. | | `parallelToolCalls?` | `boolean` | Toggle parallel tool calls. OpenAI Chat defaults on; non-chat adapters advertise only on explicit `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | Disabled-by-default downstream SSE repair for exact placeholder ids and missing terminal ids. Function-call ids are never rewritten. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries; Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is.. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries. Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | | `thinkingToggleModels?` | `string[]` | Chat models using `thinking.enabled` rather than an effort ladder. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index be60250f6..4003682d2 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -95,7 +95,7 @@ cross-route credential fallback не существует. Строки API GPT- | `noPenaltyModels?` | `string[]` | Модели, отвергающие penalty presence/frequency. | | `parallelToolCalls?` | `boolean` | Переключатель parallel tool call'ов. Для OpenAI Chat по умолчанию включено; не-chat adapter'ы рекламируют это только при явном `true`. | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | По умолчанию выключенная downstream SSE-repair для exact placeholder-id и отсутствующих terminal-id. Function-call id никогда не переписываются. | -| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста; Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | +| `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста. Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие `thinking.enabled` вместо effort-ladder. | From 70c80b4da8285fec39f8b028451684eeb287bbd5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 07:54:09 +0200 Subject: [PATCH 29/29] fix(proxy): residual 429 retry redaction, recovery parity, shared wait Redact terminal-continuation provider errors, tag failover continuation sends with the matching recovery kind, and consolidate same-target 429 body-release + wait into prepareSameTarget429Wait for core, image, and web-search paths. --- src/images/loop.ts | 16 +++---- src/lib/upstream-retry.ts | 27 ++++++++++++ src/server/responses/core.ts | 82 ++++++++++++++++++------------------ src/web-search/loop.ts | 16 +++---- tests/upstream-retry.test.ts | 42 ++++++++++++++++++ 5 files changed, 125 insertions(+), 58 deletions(-) diff --git a/src/images/loop.ts b/src/images/loop.ts index 5fbaeb038..2caa34996 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -20,7 +20,7 @@ import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, releaseResponseBodyBestEffort, sleepWithHeartbeats } from "../lib/upstream-retry"; +import { fetchWithResetRetry, prepareSameTarget429Wait } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -513,20 +513,18 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise | null; + signal?: AbortSignal; + delayMs: number; + /** + * When set, the wait yields adapter heartbeats so bridge stall watchdogs stay fed. + * Omit for pre-stream recovery paths that have no stall watchdog. + */ + heartbeatIntervalMs?: number; +} + +/** + * Shared pre-replay prep for opt-in same-target 429 waits: + * release the unread 429 body, then sleep (optionally with heartbeats). + * Callers still own attempt budgeting, abort re-checks, and the replay itself. + */ +export async function* prepareSameTarget429Wait( + options: SameTarget429WaitOptions, +): AsyncGenerator<{ type: "heartbeat" }> { + await releaseResponseBodyBestEffort(options.body, options.signal); + if (options.heartbeatIntervalMs === undefined) { + await sleepWithAbort(options.delayMs, options.signal); + return; + } + yield* sleepWithHeartbeats(options.delayMs, options.signal, options.heartbeatIntervalMs); +} + export function isConnectionResetError(err: unknown): boolean { if (!(err instanceof Error)) return false; // Aborts and timeouts are caller decisions / honest failures — never retryable. diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 3ae45806f..8b7940013 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -89,9 +89,7 @@ import { applyUpstreamRecoveryInit, fetchWithResetRetry, fetchWithTransientRetry, - releaseResponseBodyBestEffort, - sleepWithHeartbeats, - sleepWithAbort, + prepareSameTarget429Wait, } from "../../lib/upstream-retry"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { createTranslatorBudget, isTranslatorBudgetExceededError, type TranslatorBudget } from "../../lib/translator-budget"; @@ -1747,16 +1745,16 @@ async function handleResponsesInner( && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; - // Release the unread 429 body before the backoff (only the header is needed for the wait). + // Release unread body + deliberate wait via the shared same-target helper. const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - // Release the body without letting a never-settling cancel() block the abort-aware - // backoff (bounded by the abort signal and a short timeout). - await releaseResponseBodyBestEffort(upstreamResponse.body, options.abortSignal); try { - await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), - options.abortSignal, - ); + for await (const _ of prepareSameTarget429Wait({ + body: upstreamResponse.body, + signal: options.abortSignal, + delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), + })) { + // pre-stream: no stall watchdog to feed + } } catch { upstream.abort(); return clientCancelledResponse(); @@ -2649,18 +2647,16 @@ async function handleResponsesInner( && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; - // Release the unread 429 body BEFORE the backoff: only the header matters for the - // wait, and under a 429 storm the sockets would otherwise accumulate for the whole - // configured interval (same pattern as the key-failover branch below). + // Release unread body + deliberate wait via the shared same-target helper. const retryAfterHeader = upstreamResponse.headers.get("retry-after"); - // Release the body without letting a never-settling cancel() block the abort-aware - // backoff (bounded by the abort signal and a short timeout). - await releaseResponseBodyBestEffort(upstreamResponse.body, options.abortSignal); try { - await sleepWithAbort( - rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), - options.abortSignal, - ); + for await (const _ of prepareSameTarget429Wait({ + body: upstreamResponse.body, + signal: options.abortSignal, + delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), + })) { + // pre-stream: no stall watchdog to feed + } } catch { cleanupUpstreamAbort(); upstream.abort(); @@ -2805,12 +2801,15 @@ async function handleResponsesInner( */ const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator { let response: Response | undefined; + // One-shot recovery label for the next top-of-loop continuation send after a failover rotation. + let nextContinuationRecoveryKind: AttemptRecoveryKind | undefined; /** - * Build and fetch one terminal-guard continuation. `replay` marks a same-target 429 - * replay so its send records the `rate-limit-429` recovery kind; the adapter rebuild is - * deterministic for the same parsed request (tests assert byte-identical replays). + * Build and fetch one terminal-guard continuation. `recoveryKind` tags same-target and + * failover sends (`rate-limit-429`, `key-429`, `anthropic-oauth-429`, `image-413`); the + * adapter rebuild is deterministic for the same parsed request (tests assert byte-identical + * replays). */ - const fetchContinuation = async (replay = false): Promise => { + const fetchContinuation = async (recoveryKind?: AttemptRecoveryKind): Promise => { let continuationRequest: AdapterRequest | undefined; if (sameTargetRequest !== undefined && sameTargetParsed === nextParsed && sameTargetToken === transportToken) { // Same target (key/adapter/parsed/tier unchanged): replay the exact cached request. @@ -2842,8 +2841,8 @@ async function handleResponsesInner( ? builtContinuationRequest.usageLog.inputTokens : undefined; if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate; - // One label rule for continuation replays, shared by both send paths below. - const replayKind: AttemptRecoveryKind | undefined = replay ? "rate-limit-429" : undefined; + // Optional recovery label for same-target / failover continuation sends. + const replayKind: AttemptRecoveryKind | undefined = recoveryKind; try { if (activeAdapter.fetchResponse) { noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replayKind); @@ -2877,12 +2876,14 @@ async function handleResponsesInner( }; while (true) { try { - response = await fetchContinuation(); + const recoveryKind = nextContinuationRecoveryKind; + nextContinuationRecoveryKind = undefined; + response = await fetchContinuation(recoveryKind); } catch (error) { if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; } else { - yield { type: "error", message: `Provider continuation failed: ${error instanceof Error ? error.message : String(error)}` }; + yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` }; } return; } @@ -2896,20 +2897,18 @@ async function handleResponsesInner( && rateLimitRetries < rateLimitPolicy.attempts ) { rateLimitRetries += 1; - // Release the unread 429 body before the backoff (only the header is needed for the wait). + // Release unread body + heartbeat-fed wait via the shared same-target helper. const retryAfterHeader = response.headers.get("retry-after"); - // Release the body without letting a never-settling cancel() block the abort-aware - // backoff (bounded by the upstream signal and a short timeout). - await releaseResponseBodyBestEffort(response.body, upstream.signal); try { - yield* sleepWithHeartbeats( - rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), + yield* prepareSameTarget429Wait({ + body: response.body, // Listen on the upstream signal: once the SSE body is being streamed, a client // cancel aborts `upstream` through the bridge, and upstream is also linked from // options.abortSignal — so this covers both cancellation paths. - upstream.signal, - Math.min(10_000, Math.max(250, stallTimeoutMs / 2)), - ); + signal: upstream.signal, + delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()), + heartbeatIntervalMs: Math.min(10_000, Math.max(250, stallTimeoutMs / 2)), + }); } catch { if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; @@ -2925,12 +2924,12 @@ async function handleResponsesInner( return; } try { - response = await fetchContinuation(true); + response = await fetchContinuation("rate-limit-429"); } catch (error) { if (options.abortSignal?.aborted || upstream.signal.aborted) { yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; } else { - yield { type: "error", message: `Provider continuation failed: ${error instanceof Error ? error.message : String(error)}` }; + yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` }; } return; } @@ -2951,6 +2950,7 @@ async function handleResponsesInner( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire), config.cacheRetention, ); + nextContinuationRecoveryKind = "key-429"; continue; } } @@ -2981,6 +2981,7 @@ async function handleResponsesInner( config.cacheRetention, ); sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, activeAdapter.name); + nextContinuationRecoveryKind = "anthropic-oauth-429"; continue; } catch { // fall through to emit continuation error below @@ -2996,6 +2997,7 @@ async function handleResponsesInner( imageTierBias = 1; invalidateSameTargetRequest(); try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + nextContinuationRecoveryKind = "image-413"; continue; } break; diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 90a04acf6..c0eda72ce 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -8,7 +8,7 @@ import { runAnthropicWebSearch } from "./anthropic-executor"; import { clearableDeadline } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { readBoundedResponseBody } from "../lib/bounded-body"; -import { fetchWithResetRetry, releaseResponseBodyBestEffort, sleepWithHeartbeats } from "../lib/upstream-retry"; +import { fetchWithResetRetry, prepareSameTarget429Wait } from "../lib/upstream-retry"; import { rateLimitRetryDelayMs } from "../providers/key-failover"; import { isTranslatorBudgetExceededError, @@ -416,20 +416,18 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { } }); }); + + +describe("prepareSameTarget429Wait", () => { + test("releases the body then waits without heartbeats when no interval is set", async () => { + let cancelled = false; + const body = new ReadableStream({ + cancel() { + cancelled = true; + }, + }); + const events: string[] = []; + const started = Date.now(); + for await (const event of prepareSameTarget429Wait({ + body, + delayMs: 40, + })) { + events.push(event.type); + } + expect(cancelled).toBe(true); + expect(events).toEqual([]); + expect(Date.now() - started).toBeGreaterThanOrEqual(30); + }); + + test("yields heartbeats when a heartbeat interval is provided", async () => { + const body = new ReadableStream({ + cancel() { + return; + }, + }); + const events: string[] = []; + for await (const event of prepareSameTarget429Wait({ + body, + delayMs: 30, + heartbeatIntervalMs: 10, + })) { + events.push(event.type); + } + expect(events.length).toBeGreaterThanOrEqual(2); + expect(events.every(type => type === "heartbeat")).toBe(true); + }); +});