-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Bound the Codex turn await: fix unbounded hang on stalled/dead app-server + make turn budget configurable #376
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -68,6 +68,16 @@ const ROOT_DIR = path.resolve(fileURLToPath(new URL("..", import.meta.url))); | |
| const REVIEW_SCHEMA = path.join(ROOT_DIR, "schemas", "review-output.schema.json"); | ||
| const DEFAULT_STATUS_WAIT_TIMEOUT_MS = 240000; | ||
| const DEFAULT_STATUS_POLL_INTERVAL_MS = 2000; | ||
| // Full per-turn budget for background/detached jobs (no external Bash ceiling | ||
| // to collide with). codex.mjs reads CODEX_TURN_TIMEOUT_MS; this is the value we | ||
| // thread down for background runs when the user hasn't overridden it. | ||
| const DEFAULT_TURN_TIMEOUT_MS = 600000; | ||
| // Foreground runs are invoked by Claude Code's Bash tool, which SIGKILLs node | ||
| // at its own timeout (default 120000ms) and returns nothing. Set the runtime | ||
| // turn budget just below that so the foreground turn fails fast with a | ||
| // structured "turn timed out — re-run with --background" message that carries | ||
| // any partial output, instead of being killed with an empty result. | ||
| const FOREGROUND_TURN_TIMEOUT_MS = 110000; | ||
| const VALID_REASONING_EFFORTS = new Set(["none", "minimal", "low", "medium", "high", "xhigh"]); | ||
| const MODEL_ALIASES = new Map([["spark", "gpt-5.3-codex-spark"]]); | ||
| const STOP_REVIEW_TASK_MARKER = "Run a stop-gate review of the previous Claude turn."; | ||
|
|
@@ -709,9 +719,25 @@ function enqueueBackgroundTask(cwd, job, request) { | |
| }; | ||
| } | ||
|
|
||
| // Set the per-turn budget for a FOREGROUND command (codex.mjs reads | ||
| // CODEX_TURN_TIMEOUT_MS at call time). Precedence: explicit --turn-timeout-ms | ||
| // flag > a pre-set CODEX_TURN_TIMEOUT_MS env (e.g. settings.json) > the | ||
| // foreground default just under the host Bash ceiling, so a stalled turn | ||
| // returns a structured timeout instead of being SIGKILLed with no output. | ||
| // Only call this on a foreground path: a detached background worker inherits | ||
| // the parent env, so capping it here would shrink the background budget too. | ||
| function applyForegroundTurnBudget(options) { | ||
| const explicit = Number(options["turn-timeout-ms"]); | ||
| if (Number.isFinite(explicit) && explicit > 0) { | ||
| process.env.CODEX_TURN_TIMEOUT_MS = String(explicit); | ||
| } else if (!process.env.CODEX_TURN_TIMEOUT_MS) { | ||
| process.env.CODEX_TURN_TIMEOUT_MS = String(FOREGROUND_TURN_TIMEOUT_MS); | ||
| } | ||
| } | ||
|
|
||
| async function handleReviewCommand(argv, config) { | ||
| const { options, positionals } = parseCommandInput(argv, { | ||
| valueOptions: ["base", "scope", "model", "cwd"], | ||
| valueOptions: ["base", "scope", "model", "cwd", "turn-timeout-ms"], | ||
| booleanOptions: ["json", "background", "wait"], | ||
| aliasMap: { | ||
| m: "model" | ||
|
|
@@ -736,6 +762,12 @@ async function handleReviewCommand(argv, config) { | |
| jobClass: "review", | ||
| summary: metadata.summary | ||
| }); | ||
| // Review turns run foreground (--wait) through the same path as tasks; give | ||
| // them the same foreground budget so a stall returns a structured timeout | ||
| // instead of hitting the host Bash ceiling with an empty result. | ||
| if (!options.background) { | ||
| applyForegroundTurnBudget(options); | ||
| } | ||
| await runForegroundCommand( | ||
| job, | ||
| (progress) => | ||
|
|
@@ -761,7 +793,7 @@ async function handleReview(argv) { | |
|
|
||
| async function handleTask(argv) { | ||
| const { options, positionals } = parseCommandInput(argv, { | ||
| valueOptions: ["model", "effort", "cwd", "prompt-file"], | ||
| valueOptions: ["model", "effort", "cwd", "prompt-file", "turn-timeout-ms"], | ||
| booleanOptions: ["json", "write", "resume-last", "resume", "fresh", "background"], | ||
| aliasMap: { | ||
| m: "model" | ||
|
|
@@ -804,6 +836,11 @@ async function handleTask(argv) { | |
| return; | ||
| } | ||
|
|
||
| // Foreground turn budget (see applyForegroundTurnBudget). Runs only on the | ||
| // foreground path; the background branch returned above and its detached | ||
| // worker inherits the parent env unchanged (full default budget). | ||
| applyForegroundTurnBudget(options); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the stop-review gate invokes this same Useful? React with 👍 / 👎. |
||
|
|
||
| const job = buildTaskJob(workspaceRoot, taskMetadata, write); | ||
| await runForegroundCommand( | ||
| job, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,6 +51,30 @@ const DEFAULT_CONTINUE_PROMPT = | |
| const EXTERNAL_AGENT_IMPORT_COMPLETED = "externalAgentConfig/import/completed"; | ||
| const EXTERNAL_AGENT_IMPORT_TIMEOUT_MS = 2 * 60 * 1000; | ||
|
|
||
| // Hard upper bound on a single Codex turn. Without this, the completion await | ||
| // at the end of captureTurn is unbounded: it is resolved ONLY by completeTurn() | ||
| // and is never rejected on a stalled/dead process (rejectCompletion was dead | ||
| // code). The foreground budget is set below the external Bash ceiling by the | ||
| // companion so timeouts surface as structured errors instead of a SIGKILL. | ||
| const DEFAULT_TURN_TIMEOUT_MS = 600000; | ||
|
|
||
| // Resolve the per-turn budget at CALL time, not import time. The companion sets | ||
| // CODEX_TURN_TIMEOUT_MS (e.g. the foreground budget, below the Bash ceiling) | ||
| // AFTER this module is imported; reading it at import froze the value at the | ||
| // default and made --turn-timeout-ms / the foreground budget inert. Reading it | ||
| // when the turn actually starts lets the option/env override take effect. | ||
| function resolveTurnTimeoutMs(options = {}) { | ||
| const fromOptions = Number(options.turnTimeoutMs); | ||
| if (Number.isFinite(fromOptions) && fromOptions > 0) { | ||
| return fromOptions; | ||
| } | ||
| const fromEnv = Number(process.env.CODEX_TURN_TIMEOUT_MS); | ||
| if (Number.isFinite(fromEnv) && fromEnv > 0) { | ||
| return fromEnv; | ||
| } | ||
| return DEFAULT_TURN_TIMEOUT_MS; | ||
| } | ||
|
|
||
| function cleanCodexStderr(stderr) { | ||
| return stderr | ||
| .split(/\r?\n/) | ||
|
|
@@ -603,7 +627,43 @@ async function captureTurn(client, threadId, startRequest, options = {}) { | |
| completeTurn(state, response.turn); | ||
| } | ||
|
|
||
| return await state.completion; | ||
| // Bound the await so it can never outlast a dead process or a runaway turn: | ||
| // 1. state.completion — resolves on turn/completed (or inferred). Wire the | ||
| // previously-dead rejectCompletion to the client exit so an app-server | ||
| // death AFTER startRequest resolved rejects the await immediately | ||
| // instead of hanging until the deadline. Registered HERE rather than | ||
| // before startRequest: if startRequest itself rejects (e.g. broker-busy | ||
| // from turn/start, or the app-server exiting while that request is | ||
| // pending), it propagates directly and state.completion is never | ||
| // observed — wiring the exit earlier would reject an unobserved promise | ||
| // and surface as an unhandled rejection. | ||
| // 2. deadline — hard per-turn budget (resolveTurnTimeoutMs: option > | ||
| // CODEX_TURN_TIMEOUT_MS env > default, resolved at call time). | ||
| let exitRaceSettled = false; | ||
| client.exitPromise.then(() => { | ||
| if (exitRaceSettled || state.completed) { | ||
| return; | ||
| } | ||
| exitRaceSettled = true; | ||
| state.rejectCompletion( | ||
| client.exitError ?? new Error("codex app-server exited before the turn completed.") | ||
| ); | ||
| }); | ||
| const turnTimeoutMs = resolveTurnTimeoutMs(options); | ||
| let deadlineTimer = null; | ||
| const deadline = new Promise((_resolve, reject) => { | ||
| deadlineTimer = setTimeout(() => { | ||
| reject(new Error(`codex turn exceeded the ${turnTimeoutMs}ms turn budget.`)); | ||
| }, turnTimeoutMs); | ||
|
Comment on lines
+655
to
+657
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the new deadline fires, this rejects out of Useful? React with 👍 / 👎. |
||
| deadlineTimer.unref?.(); | ||
| }); | ||
| try { | ||
| return await Promise.race([state.completion, deadline]); | ||
| } finally { | ||
| if (deadlineTimer) { | ||
| clearTimeout(deadlineTimer); | ||
| } | ||
| } | ||
| } finally { | ||
| clearCompletionTimer(state); | ||
| client.setNotificationHandler(previousHandler ?? null); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Accepting
--turn-timeout-msfortaskmakestask --background --turn-timeout-ms 3000look supported, but the background branch returns beforeapplyForegroundTurnBudget()andbuildTaskRequest()only stores cwd/model/effort/prompt/write/resumeLast/jobId, so the detachedtask-workerinherits no CLI override and falls back toCODEX_TURN_TIMEOUT_MSor the 600000ms default. Users trying to bound a background turn will still get the default budget unless they set the environment variable manually.Useful? React with 👍 / 👎.