Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 32 additions & 18 deletions src/server/responses/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,11 @@ async function retryCodexPoolOnAlternateAccount(
firstAuthCtx.writerGeneration,
);
}
if (!shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) {
const deferFirstOutcome = shouldDeferCodexResetDerivedCooldown(
firstResponse,
options.deferCodexResetDerivedCooldown,
);
const recordFirstOutcome = (): void => {
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
...quotaMeta,
threadId: req.headers.get("x-codex-parent-thread-id"),
Expand All @@ -446,8 +450,10 @@ async function retryCodexPoolOnAlternateAccount(
// Retry already advanced the RR ring via excludeAccountId — reuse for promotion.
...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}),
});
}

};
// Only a combo reset-derived outcome is deferred. Retry-After, defaults, and
// ordinary requests must block the first account before the alternate send.
if (!deferFirstOutcome) recordFirstOutcome();
const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx);
const retryProvider = applyCodexAuthContextToProvider(
stripCodexRuntimeProviderFields(route.provider),
Expand All @@ -474,8 +480,9 @@ async function retryCodexPoolOnAlternateAccount(
);

noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
let upstreamResponse: Response;
try {
const upstreamResponse = await fetchWithHeaderTimeout(
upstreamResponse = await fetchWithHeaderTimeout(
request.url,
{
method: request.method,
Expand All @@ -490,26 +497,33 @@ async function retryCodexPoolOnAlternateAccount(
// dead-host rejection after the credential was seen (#914).
route.provider.authMode === "forward",
);
// A real HTTP response proves the host was reached (#914).
const retryHostKey = upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url));
if (normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0) {
resetUpstreamHostHealth(retryHostKey, null);
} else {
resetUpstreamHostHealth(retryHostKey);
}
return {
kind: "retried",
authCtx: retryAuthCtx,
request,
upstreamResponse,
selectedForwardHeaders: retryHeaders,
};
} catch (error) {
// Attribute the transport failure to the alternate account (already selected).
return { kind: "transport", error, authCtx: retryAuthCtx };
} finally {
request.releaseBodyObservation?.();
}
// A real HTTP response proves the host was reached (#914).
const retryHostKey = upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url));
if (normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0) {
resetUpstreamHostHealth(retryHostKey, null);
} else {
resetUpstreamHostHealth(retryHostKey);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if (deferFirstOutcome && upstreamResponse.ok) {
// Deferral keeps the first account eligible for a later combo model while an
// alternate attempt is still fallible. Commit its quota outcome only once the
// alternate account returns a successful HTTP response; otherwise the combo may
// still need the first account for its next target.
recordFirstOutcome();
}
return {
kind: "retried",
authCtx: retryAuthCtx,
request,
upstreamResponse,
selectedForwardHeaders: retryHeaders,
};
}


Expand Down
110 changes: 110 additions & 0 deletions tests/server-auth.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
CODEX_THREAD_AFFINITY_IDLE_TTL_MS,
clearCodexUpstreamHealth,
clearThreadAccountMap,
getCodexQuotaHealthSnapshot,
getCodexUpstreamHealth,
isCodexAccountSoftAvoided,
recordCodexUpstreamOutcome,
Expand Down Expand Up @@ -194,6 +195,7 @@ async function startPoolRetryHarness(
pausedAccountIds?: string[];
reauthAccountIds?: string[];
omitCredentialAccountIds?: string[];
combos?: OcxConfig["combos"];
} = {},
): Promise<PoolRetryHarness> {
await removeTestDirBestEffort(TEST_DIR);
Expand Down Expand Up @@ -250,6 +252,7 @@ async function startPoolRetryHarness(
...(options.visionSidecarModel ? { visionSidecar: { model: options.visionSidecarModel } } : {}),
...(options.websockets ? { websockets: true } : {}),
...(options.streamMode ? { streamMode: options.streamMode } : {}),
...(options.combos ? { combos: options.combos } : {}),
} as OcxConfig;
saveConfig(config);
if (!options.omitCredentialAccountIds?.includes("pool-a")) {
Expand Down Expand Up @@ -2318,6 +2321,113 @@ describe("server local API auth", () => {
}
});

test("#584: Retry-After cools the first account even when its account retry fails", async () => {
const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a"
? new Response(JSON.stringify({ error: { message: "rate limited" } }), {
status: 429,
headers: { "content-type": "application/json", "retry-after": "60" },
})
: new Response(JSON.stringify({ error: { message: "upstream unavailable" } }), {
status: 503,
headers: { "content-type": "application/json" },
}));
try {
const response = await harness.request();
expect(response.status).toBe(503);
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]);
const health = getCodexUpstreamHealth("pool-a");
expect(health).toMatchObject({
cooldownSource: "retry-after",
});
expect(health?.cooldownUntil).toBeGreaterThan(Date.now());
} finally {
await stopPoolRetryHarness(harness);
}
}, { timeout: SERVER_BUDGET_MS });
Comment thread
luvs01 marked this conversation as resolved.

test("combo reset deferral still cools the first account when its account retry succeeds", async () => {
const harness = await startPoolRetryHarness(
accountId => accountId === "acct-pool-a"
? new Response(JSON.stringify({ error: { message: "rate limited" } }), {
status: 429,
headers: {
"content-type": "application/json",
"x-codex-primary-reset-at": String(Math.floor(Date.now() / 1000) + 3600),
},
})
: Response.json({ id: "combo-account-retry-success", status: "completed", output: [] }),
{
combos: {
quota: {
strategy: "failover",
targets: [
{ provider: "openai", model: POOL_RETRY_MODEL },
{ provider: "openai", model: `${POOL_RETRY_MODEL}-fallback` },
],
},
},
},
);
try {
const response = await harness.request({ model: "combo/quota" });
expect(response.status).toBe(200);
expect((await response.json() as { id: string }).id).toBe("combo-account-retry-success");
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]);
expect(getCodexQuotaHealthSnapshot("pool-a", "shared")).toMatchObject({
cooldownUntil: expect.any(Number),
cooldownSource: "reset-derived",
quotaScope: "shared",
});
expect(loadConfig().activeCodexAccountId).toBe("pool-b");
} finally {
await stopPoolRetryHarness(harness);
}
}, { timeout: SERVER_BUDGET_MS });

test("combo reset deferral preserves the first account when its account retry also fails", async () => {
const harness = await startPoolRetryHarness(
async (accountId, request) => {
const body = await request.json() as { model?: string };
if (body.model === `${POOL_RETRY_MODEL}-fallback`) {
return Response.json({ id: "combo-later-model-success", status: "completed", output: [] });
}
if (accountId === "acct-pool-a") {
return new Response(JSON.stringify({ error: { message: "rate limited" } }), {
status: 429,
headers: {
"content-type": "application/json",
"x-codex-primary-reset-at": String(Math.floor(Date.now() / 1000) + 3600),
},
});
}
return new Response(JSON.stringify({ error: { message: "retry later" } }), {
status: 429,
headers: { "content-type": "application/json", "retry-after": "60" },
});
},
{
combos: {
quota: {
strategy: "failover",
targets: [
{ provider: "openai", model: POOL_RETRY_MODEL },
{ provider: "openai", model: `${POOL_RETRY_MODEL}-fallback` },
],
},
},
},
);
try {
const response = await harness.request({ model: "combo/quota" });
expect(response.status).toBe(200);
expect((await response.json() as { id: string }).id).toBe("combo-later-model-success");
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b", "acct-pool-a"]);
expect(getCodexQuotaHealthSnapshot("pool-a", "shared")).toBeNull();
} finally {
await stopPoolRetryHarness(harness);
}
Comment thread
luvs01 marked this conversation as resolved.
}, { timeout: SERVER_BUDGET_MS });

test("#584: pre-stream 429 with one eligible account preserves the original 429", async () => {
const body = JSON.stringify({ error: { message: "rate limited" } });
const harness = await startPoolRetryHarness(
Expand Down
Loading