From a05d74e2b7ad0d707fbe83301a7693a2ca7a04cd Mon Sep 17 00:00:00 2001 From: hsryu Date: Thu, 18 Jun 2026 16:07:48 +0900 Subject: [PATCH 01/20] refactor(state): extract resolveStateRoot for cwd-independent state-root access --- plugins/codex/scripts/lib/state.mjs | 8 ++++++-- tests/broker-lifecycle.test.mjs | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/broker-lifecycle.test.mjs diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 2da23498f..5f8e04b8a 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -26,6 +26,11 @@ function defaultState() { }; } +export function resolveStateRoot() { + const pluginDataDir = process.env[PLUGIN_DATA_ENV]; + return pluginDataDir ? path.join(pluginDataDir, "state") : FALLBACK_STATE_ROOT_DIR; +} + export function resolveStateDir(cwd) { const workspaceRoot = resolveWorkspaceRoot(cwd); let canonicalWorkspaceRoot = workspaceRoot; @@ -38,8 +43,7 @@ export function resolveStateDir(cwd) { const slugSource = path.basename(workspaceRoot) || "workspace"; const slug = slugSource.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "workspace"; const hash = createHash("sha256").update(canonicalWorkspaceRoot).digest("hex").slice(0, 16); - const pluginDataDir = process.env[PLUGIN_DATA_ENV]; - const stateRoot = pluginDataDir ? path.join(pluginDataDir, "state") : FALLBACK_STATE_ROOT_DIR; + const stateRoot = resolveStateRoot(); return path.join(stateRoot, `${slug}-${hash}`); } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs new file mode 100644 index 000000000..95aec6348 --- /dev/null +++ b/tests/broker-lifecycle.test.mjs @@ -0,0 +1,29 @@ +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import assert from "node:assert/strict"; + +import { makeTempDir } from "./helpers.mjs"; +import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; + +test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { + const pluginData = makeTempDir(); + const prev = process.env.CLAUDE_PLUGIN_DATA; + process.env.CLAUDE_PLUGIN_DATA = pluginData; + try { + assert.equal(resolveStateRoot(), path.join(pluginData, "state")); + } finally { + if (prev == null) delete process.env.CLAUDE_PLUGIN_DATA; + else process.env.CLAUDE_PLUGIN_DATA = prev; + } +}); + +test("resolveStateRoot falls back to a tmp dir when CLAUDE_PLUGIN_DATA is unset", () => { + const prev = process.env.CLAUDE_PLUGIN_DATA; + delete process.env.CLAUDE_PLUGIN_DATA; + try { + assert.equal(resolveStateRoot(), path.join(os.tmpdir(), "codex-companion")); + } finally { + if (prev != null) process.env.CLAUDE_PLUGIN_DATA = prev; + } +}); From 615cd92e4f30afc8713a7f982bd4fd6f980ff1ba Mon Sep 17 00:00:00 2001 From: hsryu Date: Thu, 18 Jun 2026 16:11:10 +0900 Subject: [PATCH 02/20] feat(broker): record owning sessionId in broker.json on creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 브로커 생성 시 CODEX_COMPANION_SESSION_ID를 broker.json에 sessionId 필드로 기록. resolveSessionId(options)는 options.sessionId → options.env → process.env → null 순서로 해소. 기존 세션 재사용 경로(early-return)는 변경 없음. Co-Authored-By: Claude Sonnet 4.6 --- .../codex/scripts/lib/broker-lifecycle.mjs | 12 ++++++++- tests/broker-lifecycle.test.mjs | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ef763819c..d21ba69a0 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -11,6 +11,15 @@ import { resolveStateDir } from "./state.mjs"; export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; +const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; + +export function resolveSessionId(options = {}) { + if (options.sessionId) { + return options.sessionId; + } + const env = options.env ?? process.env; + return env[SESSION_ID_ENV] ?? null; +} export function createBrokerSessionDir(prefix = "cxc-") { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -164,7 +173,8 @@ export async function ensureBrokerSession(cwd, options = {}) { pidFile, logFile, sessionDir, - pid: child.pid ?? null + pid: child.pid ?? null, + sessionId: resolveSessionId(options) }; saveBrokerSession(cwd, session); return session; diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 95aec6348..73e9739e7 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -5,6 +5,7 @@ import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; +import { resolveSessionId } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { const pluginData = makeTempDir(); @@ -27,3 +28,29 @@ test("resolveStateRoot falls back to a tmp dir when CLAUDE_PLUGIN_DATA is unset" if (prev != null) process.env.CLAUDE_PLUGIN_DATA = prev; } }); + +test("resolveSessionId prefers explicit option, then env, then null", () => { + assert.equal(resolveSessionId({ sessionId: "explicit" }), "explicit"); + assert.equal(resolveSessionId({ env: { CODEX_COMPANION_SESSION_ID: "from-env" } }), "from-env"); +}); + +test("resolveSessionId reads process.env when no option/env given", () => { + const prev = process.env.CODEX_COMPANION_SESSION_ID; + process.env.CODEX_COMPANION_SESSION_ID = "proc-env"; + try { + assert.equal(resolveSessionId({}), "proc-env"); + } finally { + if (prev == null) delete process.env.CODEX_COMPANION_SESSION_ID; + else process.env.CODEX_COMPANION_SESSION_ID = prev; + } +}); + +test("resolveSessionId returns null when nothing is set", () => { + const prev = process.env.CODEX_COMPANION_SESSION_ID; + delete process.env.CODEX_COMPANION_SESSION_ID; + try { + assert.equal(resolveSessionId({ env: {} }), null); + } finally { + if (prev != null) process.env.CODEX_COMPANION_SESSION_ID = prev; + } +}); From 6a1bca368b605072715178031a42576fa2627224 Mon Sep 17 00:00:00 2001 From: hsryu Date: Thu, 18 Jun 2026 16:14:41 +0900 Subject: [PATCH 03/20] feat(broker): add teardownBrokersForSession for cwd-independent cleanup --- .../codex/scripts/lib/broker-lifecycle.mjs | 47 ++++++++++- tests/broker-lifecycle.test.mjs | 77 ++++++++++++++++++- 2 files changed, 121 insertions(+), 3 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index d21ba69a0..aba0d465a 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -6,7 +6,7 @@ import process from "node:process"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { resolveStateDir } from "./state.mjs"; +import { resolveStateDir, resolveStateRoot } from "./state.mjs"; export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; @@ -180,6 +180,51 @@ export async function ensureBrokerSession(cwd, options = {}) { return session; } +export async function teardownBrokersForSession(sessionId, { killProcess = null } = {}) { + if (!sessionId) { + return 0; + } + const stateRoot = resolveStateRoot(); + if (!fs.existsSync(stateRoot)) { + return 0; + } + + let count = 0; + for (const entry of fs.readdirSync(stateRoot)) { + const stateFile = path.join(stateRoot, entry, BROKER_STATE_FILE); + if (!fs.existsSync(stateFile)) { + continue; + } + + let session; + try { + session = JSON.parse(fs.readFileSync(stateFile, "utf8")); + } catch { + continue; + } + if (!session || session.sessionId !== sessionId) { + continue; + } + + if (session.endpoint) { + await sendBrokerShutdown(session.endpoint); + } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + count += 1; + } + return count; +} + export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { if (Number.isFinite(pid) && killProcess) { try { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 73e9739e7..7480da288 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -1,11 +1,12 @@ +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; -import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; -import { resolveSessionId } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { resolveStateRoot, resolveStateRoot as stateRootForTest } from "../plugins/codex/scripts/lib/state.mjs"; +import { resolveSessionId, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { const pluginData = makeTempDir(); @@ -54,3 +55,75 @@ test("resolveSessionId returns null when nothing is set", () => { if (prev != null) process.env.CODEX_COMPANION_SESSION_ID = prev; } }); + +function writeBrokerJson(stateRoot, dirName, session) { + const dir = path.join(stateRoot, dirName); + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, "broker.json"); + fs.writeFileSync(file, JSON.stringify(session), "utf8"); + return file; +} + +function withPluginData(fn) { + const pluginData = makeTempDir(); + const prev = process.env.CLAUDE_PLUGIN_DATA; + process.env.CLAUDE_PLUGIN_DATA = pluginData; + return Promise.resolve(fn(pluginData)).finally(() => { + if (prev == null) delete process.env.CLAUDE_PLUGIN_DATA; + else process.env.CLAUDE_PLUGIN_DATA = prev; + }); +} + +test("teardownBrokersForSession tears down a broker registered for a different cwd", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const sessionDir = makeTempDir(); + const pidFile = path.join(sessionDir, "broker.pid"); + const logFile = path.join(sessionDir, "broker.log"); + fs.writeFileSync(pidFile, "12345\n"); + fs.writeFileSync(logFile, ""); + const brokerJson = writeBrokerJson(stateRoot, "worktree-deadbeefdeadbeef", { + endpoint: "unix:/tmp/codex-test-nonexistent.sock", + pidFile, logFile, sessionDir, pid: 12345, sessionId: "S" + }); + + const killed = []; + const count = await teardownBrokersForSession("S", { killProcess: (pid) => killed.push(pid) }); + + assert.equal(count, 1); + assert.deepEqual(killed, [12345]); + assert.equal(fs.existsSync(brokerJson), false); + }); +}); + +test("teardownBrokersForSession leaves non-matching sessionId brokers intact", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "other-1111111111111111", { + endpoint: "unix:/tmp/codex-test-nonexistent2.sock", + pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" + }); + const count = await teardownBrokersForSession("OTHER", { killProcess: () => {} }); + assert.equal(count, 0); + assert.equal(fs.existsSync(brokerJson), true); + }); +}); + +test("teardownBrokersForSession ignores broker.json without sessionId (legacy)", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "legacy-2222222222222222", { + endpoint: "unix:/tmp/codex-test-nonexistent3.sock", pid: null + }); + const count = await teardownBrokersForSession("S", { killProcess: () => {} }); + assert.equal(count, 0); + assert.equal(fs.existsSync(brokerJson), true); + }); +}); + +test("teardownBrokersForSession is a no-op for empty sessionId", async () => { + await withPluginData(async () => { + const count = await teardownBrokersForSession("", { killProcess: () => {} }); + assert.equal(count, 0); + }); +}); From 8aa56c3f48f07c2fea2e3333640c59942a14cdc6 Mon Sep 17 00:00:00 2001 From: hsryu Date: Thu, 18 Jun 2026 16:19:45 +0900 Subject: [PATCH 04/20] fix(session): tear down brokers by sessionId on SessionEnd (#380) --- .../codex/scripts/session-lifecycle-hook.mjs | 23 +++++++++++++----- tests/broker-lifecycle.test.mjs | 24 ++++++++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 9655eaef4..a5e963d79 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -11,7 +11,8 @@ import { loadBrokerSession, PID_FILE_ENV, sendBrokerShutdown, - teardownBrokerSession + teardownBrokerSession, + teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; @@ -78,7 +79,7 @@ function handleSessionStart(input) { appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); } -async function handleSessionEnd(input) { +export async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); const brokerSession = loadBrokerSession(cwd) ?? @@ -109,6 +110,11 @@ async function handleSessionEnd(input) { killProcess: terminateProcessTree }); clearBrokerSession(cwd); + + const sessionId = input.session_id || process.env[SESSION_ID_ENV]; + if (sessionId) { + await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); + } } async function main() { @@ -125,7 +131,12 @@ async function main() { } } -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); +// Only run main() when executed directly (not when imported by tests). +const isMain = process.argv[1] && + new URL(import.meta.url).pathname === new URL(process.argv[1], import.meta.url).pathname; +if (isMain) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 7480da288..35c5bf949 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -5,8 +5,11 @@ import test from "node:test"; import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; -import { resolveStateRoot, resolveStateRoot as stateRootForTest } from "../plugins/codex/scripts/lib/state.mjs"; +import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; import { resolveSessionId, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; + +const stateRootForTest = resolveStateRoot; test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { const pluginData = makeTempDir(); @@ -127,3 +130,22 @@ test("teardownBrokersForSession is a no-op for empty sessionId", async () => { assert.equal(count, 0); }); }); + +test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const sessionDir = makeTempDir(); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999999\n"); // non-existent pid, harmless to signal + const brokerJson = writeBrokerJson(stateRoot, "worktree-33333333deadbeef", { + endpoint: "unix:/tmp/codex-test-nonexistent4.sock", + pidFile, logFile: null, sessionDir, pid: 999999999, sessionId: "S" + }); + + // cwd is a DIFFERENT path than the broker's workspace — the cwd-based path + // would miss; the session-based path must still tear it down. + await handleSessionEnd({ cwd: makeTempDir(), session_id: "S" }); + + assert.equal(fs.existsSync(brokerJson), false); + }); +}); From 82a372fc1fa3c0ec409d949762be5e0aee73962d Mon Sep 17 00:00:00 2001 From: hsryu Date: Thu, 18 Jun 2026 18:22:44 +0900 Subject: [PATCH 05/20] Keep session hooks running through symlinked plugin roots Compare the hook entrypoint and argv path after resolving realpaths so Node's import.meta.url canonicalization does not bypass direct execution when CLAUDE_PLUGIN_ROOT is a symlink. Add a runtime regression test that invokes SessionStart through a symlinked plugin root and verifies the exported session environment. Constraint: Node resolves imported module URLs to real paths while argv preserves the invoked symlink path. Rejected: Raw URL pathname comparison | fails for symlinked plugin installations. Confidence: high Scope-risk: narrow Directive: Keep hook direct-execution checks symlink-aware; SessionEnd cleanup depends on main() running from installed hook commands. Tested: node --test tests/runtime.test.mjs tests/broker-lifecycle.test.mjs; npm test Not-tested: npm run build fails before this change because generated app-server types do not match current source types. --- .../codex/scripts/session-lifecycle-hook.mjs | 14 ++++++-- tests/runtime.test.mjs | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index a5e963d79..23834727e 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -1,7 +1,9 @@ #!/usr/bin/env node import fs from "node:fs"; +import path from "node:path"; import process from "node:process"; +import { fileURLToPath } from "node:url"; import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; @@ -131,9 +133,17 @@ async function main() { } } +function isExecutedDirectly() { + if (!process.argv[1]) { + return false; + } + + return fs.realpathSync(fileURLToPath(import.meta.url)) === + fs.realpathSync(path.resolve(process.argv[1])); +} + // Only run main() when executed directly (not when imported by tests). -const isMain = process.argv[1] && - new URL(import.meta.url).pathname === new URL(process.argv[1], import.meta.url).pathname; +const isMain = isExecutedDirectly(); if (isMain) { main().catch((error) => { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 90408372f..8447dd15c 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -562,6 +562,38 @@ test("session start hook exports the Claude session id and plugin data dir for l ); }); +test("session start hook runs when invoked through a symlinked plugin root", () => { + const repo = makeTempDir(); + const linkParent = makeTempDir(); + const pluginLink = path.join(linkParent, "codex-link"); + fs.symlinkSync(PLUGIN_ROOT, pluginLink, process.platform === "win32" ? "junction" : "dir"); + + const envFile = path.join(makeTempDir(), "claude-env.sh"); + fs.writeFileSync(envFile, "", "utf8"); + const pluginDataDir = makeTempDir(); + const symlinkedSessionHook = path.join(pluginLink, "scripts", "session-lifecycle-hook.mjs"); + + const result = run("node", [symlinkedSessionHook, "SessionStart"], { + cwd: repo, + env: { + ...process.env, + CLAUDE_ENV_FILE: envFile, + CLAUDE_PLUGIN_DATA: pluginDataDir + }, + input: JSON.stringify({ + hook_event_name: "SessionStart", + session_id: "sess-symlink", + cwd: repo + }) + }); + + assert.equal(result.status, 0, result.stderr); + assert.equal( + fs.readFileSync(envFile, "utf8"), + `export CODEX_COMPANION_SESSION_ID='sess-symlink'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` + ); +}); + test("write task output focuses on the Codex result without generic follow-up hints", () => { const repo = makeTempDir(); const binDir = makeTempDir(); From a897e88f5cf020d2c90b0c05a20deae17219d8a1 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Thu, 18 Jun 2026 19:52:52 +0900 Subject: [PATCH 06/20] fix(broker): preserve shared broker ownership across sessions --- .../codex/scripts/lib/broker-lifecycle.mjs | 53 ++++++++- .../codex/scripts/session-lifecycle-hook.mjs | 16 +-- tests/broker-lifecycle.test.mjs | 102 +++++++++++++++++- 3 files changed, 160 insertions(+), 11 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index aba0d465a..027c8e263 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -21,6 +21,36 @@ export function resolveSessionId(options = {}) { return env[SESSION_ID_ENV] ?? null; } +function brokerSessionOwners(session) { + const owners = []; + if (Array.isArray(session?.sessionIds)) { + owners.push(...session.sessionIds); + } + if (session?.sessionId) { + owners.push(session.sessionId); + } + return [...new Set(owners.filter(Boolean))]; +} + +export function hasBrokerSessionOwners(session) { + return brokerSessionOwners(session).length > 0; +} + +function withBrokerSessionOwner(session, sessionId) { + if (!sessionId) { + return session; + } + const owners = brokerSessionOwners(session); + if (!owners.includes(sessionId)) { + owners.push(sessionId); + } + return { + ...session, + sessionId: owners[0] ?? sessionId, + sessionIds: owners + }; +} + export function createBrokerSessionDir(prefix = "cxc-") { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); } @@ -122,7 +152,11 @@ async function isBrokerEndpointReady(endpoint) { export async function ensureBrokerSession(cwd, options = {}) { const existing = loadBrokerSession(cwd); if (existing && (await isBrokerEndpointReady(existing.endpoint))) { - return existing; + const withOwner = withBrokerSessionOwner(existing, resolveSessionId(options)); + if (withOwner !== existing) { + saveBrokerSession(cwd, withOwner); + } + return withOwner; } if (existing) { @@ -176,8 +210,9 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: child.pid ?? null, sessionId: resolveSessionId(options) }; - saveBrokerSession(cwd, session); - return session; + const withOwner = withBrokerSessionOwner(session, session.sessionId); + saveBrokerSession(cwd, withOwner); + return withOwner; } export async function teardownBrokersForSession(sessionId, { killProcess = null } = {}) { @@ -202,7 +237,17 @@ export async function teardownBrokersForSession(sessionId, { killProcess = null } catch { continue; } - if (!session || session.sessionId !== sessionId) { + const owners = brokerSessionOwners(session); + if (!owners.includes(sessionId)) { + continue; + } + const remainingOwners = owners.filter((owner) => owner !== sessionId); + if (remainingOwners.length > 0) { + fs.writeFileSync( + stateFile, + `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, + "utf8" + ); continue; } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 23834727e..037ed43db 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -9,6 +9,7 @@ import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { clearBrokerSession, + hasBrokerSessionOwners, LOG_FILE_ENV, loadBrokerSession, PID_FILE_ENV, @@ -83,6 +84,12 @@ function handleSessionStart(input) { export async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); + const sessionId = input.session_id || process.env[SESSION_ID_ENV]; + cleanupSessionJobs(cwd, sessionId); + if (sessionId) { + await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); + } + const brokerSession = loadBrokerSession(cwd) ?? (process.env[BROKER_ENDPOINT_ENV] @@ -92,6 +99,9 @@ export async function handleSessionEnd(input) { logFile: process.env[LOG_FILE_ENV] ?? null } : null); + if (sessionId && hasBrokerSessionOwners(brokerSession)) { + return; + } const brokerEndpoint = brokerSession?.endpoint ?? null; const pidFile = brokerSession?.pidFile ?? null; const logFile = brokerSession?.logFile ?? null; @@ -102,7 +112,6 @@ export async function handleSessionEnd(input) { await sendBrokerShutdown(brokerEndpoint); } - cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); teardownBrokerSession({ endpoint: brokerEndpoint, pidFile, @@ -112,11 +121,6 @@ export async function handleSessionEnd(input) { killProcess: terminateProcessTree }); clearBrokerSession(cwd); - - const sessionId = input.session_id || process.env[SESSION_ID_ENV]; - if (sessionId) { - await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); - } } async function main() { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 35c5bf949..2f5e42915 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -1,12 +1,20 @@ import fs from "node:fs"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; +import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; -import { resolveSessionId, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { + ensureBrokerSession, + loadBrokerSession, + resolveSessionId, + saveBrokerSession, + teardownBrokersForSession +} from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; const stateRootForTest = resolveStateRoot; @@ -77,6 +85,35 @@ function withPluginData(fn) { }); } +async function withReadyBroker(fn) { + const sessionDir = makeTempDir(); + const endpoint = createBrokerEndpoint(sessionDir); + const target = parseBrokerEndpoint(endpoint); + const requests = []; + const server = net.createServer((socket) => { + socket.setEncoding("utf8"); + socket.on("data", (chunk) => { + requests.push(chunk); + socket.write(`${JSON.stringify({ id: 1, result: {} })}\n`); + socket.end(); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(target.path, () => { + server.off("error", reject); + resolve(); + }); + }); + + try { + return await fn({ endpoint, requests, sessionDir }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + test("teardownBrokersForSession tears down a broker registered for a different cwd", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); @@ -131,6 +168,69 @@ test("teardownBrokersForSession is a no-op for empty sessionId", async () => { }); }); +test("reusing a ready broker transfers cleanup ownership to the later session", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + const pidFile = path.join(sessionDir, "broker.pid"); + const logFile = path.join(sessionDir, "broker.log"); + fs.writeFileSync(pidFile, "12345\n"); + fs.writeFileSync(logFile, ""); + saveBrokerSession(cwd, { + endpoint, + pidFile, + logFile, + sessionDir, + pid: 12345, + sessionId: "A" + }); + + const reused = await ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "B" } }); + assert.equal(reused.endpoint, endpoint); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["A", "B"]); + + const killed = []; + assert.equal(await teardownBrokersForSession("A", { killProcess: (pid) => killed.push(pid) }), 0); + assert.deepEqual(killed, []); + assert.equal(loadBrokerSession(cwd).sessionId, "B"); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["B"]); + assert.equal(requests.length, 0); + + assert.equal(await teardownBrokersForSession("B", { killProcess: (pid) => killed.push(pid) }), 1); + assert.deepEqual(killed, [12345]); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(requests.length, 1); + }); + }); +}); + +test("handleSessionEnd removes only the ending owner from a shared cwd broker", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + const pidFile = path.join(sessionDir, "broker.pid"); + const logFile = path.join(sessionDir, "broker.log"); + fs.writeFileSync(pidFile, "12345\n"); + fs.writeFileSync(logFile, ""); + saveBrokerSession(cwd, { + endpoint, + pidFile, + logFile, + sessionDir, + pid: 12345, + sessionId: "A", + sessionIds: ["A", "B"] + }); + + await handleSessionEnd({ cwd, session_id: "A" }); + + assert.equal(loadBrokerSession(cwd).sessionId, "B"); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["B"]); + assert.equal(requests.length, 0); + }); + }); +}); + test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); From c22bb77eab87c3586e0e20381451c8596b69734d Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Tue, 7 Jul 2026 18:33:12 +0900 Subject: [PATCH 07/20] fix(broker): serialize session owner teardown Serialize broker owner updates with a per-state-file lock so concurrent SessionEnd hooks cannot leave an already-ended owner behind. Ensure broker teardown still runs when job cleanup throws, and bound broker shutdown RPC waits so unresponsive endpoints cannot hang SessionEnd. Add regressions for each path. --- .../codex/scripts/lib/broker-lifecycle.mjs | 158 ++++++++++----- .../codex/scripts/session-lifecycle-hook.mjs | 14 +- tests/broker-lifecycle.test.mjs | 181 +++++++++++++++++- 3 files changed, 308 insertions(+), 45 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 027c8e263..32d2e8fe5 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -12,6 +12,9 @@ export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +const BROKER_STATE_LOCK_STALE_MS = 30000; +const BROKER_STATE_LOCK_TIMEOUT_MS = 5000; +const BROKER_SHUTDOWN_TIMEOUT_MS = 1000; export function resolveSessionId(options = {}) { if (options.sessionId) { @@ -51,6 +54,47 @@ function withBrokerSessionOwner(session, sessionId) { }; } +async function sleep(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function withBrokerStateFileLock(stateFile, fn, options = {}) { + const lockDir = `${stateFile}.lock`; + const timeoutMs = options.timeoutMs ?? BROKER_STATE_LOCK_TIMEOUT_MS; + const staleMs = options.staleMs ?? BROKER_STATE_LOCK_STALE_MS; + const deadline = Date.now() + timeoutMs; + + while (true) { + try { + fs.mkdirSync(lockDir); + break; + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + try { + const stat = fs.statSync(lockDir); + if (Date.now() - stat.mtimeMs > staleMs) { + fs.rmSync(lockDir, { recursive: true, force: true }); + continue; + } + } catch { + continue; + } + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for broker state lock: ${stateFile}`); + } + await sleep(25); + } + } + + try { + return await fn(); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } +} + export function createBrokerSessionDir(prefix = "cxc-") { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); } @@ -79,19 +123,37 @@ export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { return false; } -export async function sendBrokerShutdown(endpoint) { +export async function sendBrokerShutdown(endpoint, { timeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS } = {}) { await new Promise((resolve) => { + let settled = false; + let timer = null; const socket = connectToEndpoint(endpoint); + const finish = () => { + if (settled) { + return; + } + settled = true; + if (timer) { + clearTimeout(timer); + } + resolve(); + }; + if (timeoutMs > 0) { + timer = setTimeout(() => { + socket.destroy(); + finish(); + }, timeoutMs); + } socket.setEncoding("utf8"); socket.on("connect", () => { socket.write(`${JSON.stringify({ id: 1, method: "broker/shutdown", params: {} })}\n`); }); socket.on("data", () => { socket.end(); - resolve(); + finish(); }); - socket.on("error", resolve); - socket.on("close", resolve); + socket.on("error", finish); + socket.on("close", finish); }); } @@ -152,11 +214,15 @@ async function isBrokerEndpointReady(endpoint) { export async function ensureBrokerSession(cwd, options = {}) { const existing = loadBrokerSession(cwd); if (existing && (await isBrokerEndpointReady(existing.endpoint))) { - const withOwner = withBrokerSessionOwner(existing, resolveSessionId(options)); - if (withOwner !== existing) { - saveBrokerSession(cwd, withOwner); - } - return withOwner; + const stateFile = resolveBrokerStateFile(cwd); + return await withBrokerStateFileLock(stateFile, () => { + const current = loadBrokerSession(cwd) ?? existing; + const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); + if (withOwner !== current) { + saveBrokerSession(cwd, withOwner); + } + return withOwner; + }); } if (existing) { @@ -215,7 +281,7 @@ export async function ensureBrokerSession(cwd, options = {}) { return withOwner; } -export async function teardownBrokersForSession(sessionId, { killProcess = null } = {}) { +export async function teardownBrokersForSession(sessionId, { killProcess = null, shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS } = {}) { if (!sessionId) { return 0; } @@ -231,41 +297,47 @@ export async function teardownBrokersForSession(sessionId, { killProcess = null continue; } - let session; - try { - session = JSON.parse(fs.readFileSync(stateFile, "utf8")); - } catch { - continue; - } - const owners = brokerSessionOwners(session); - if (!owners.includes(sessionId)) { - continue; - } - const remainingOwners = owners.filter((owner) => owner !== sessionId); - if (remainingOwners.length > 0) { - fs.writeFileSync( - stateFile, - `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, - "utf8" - ); - continue; - } + await withBrokerStateFileLock(stateFile, async () => { + if (!fs.existsSync(stateFile)) { + return; + } - if (session.endpoint) { - await sendBrokerShutdown(session.endpoint); - } - teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, - killProcess + let session; + try { + session = JSON.parse(fs.readFileSync(stateFile, "utf8")); + } catch { + return; + } + const owners = brokerSessionOwners(session); + if (!owners.includes(sessionId)) { + return; + } + const remainingOwners = owners.filter((owner) => owner !== sessionId); + if (remainingOwners.length > 0) { + fs.writeFileSync( + stateFile, + `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, + "utf8" + ); + return; + } + + if (session.endpoint) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); + } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + count += 1; }); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); - } - count += 1; } return count; } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index dd3fcf857..ddabc37fe 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -87,7 +87,13 @@ function handleSessionStart(input) { export async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); const sessionId = input.session_id || process.env[SESSION_ID_ENV]; - cleanupSessionJobs(cwd, sessionId); + let cleanupError = null; + try { + cleanupSessionJobs(cwd, sessionId); + } catch (error) { + cleanupError = error; + } + if (sessionId) { await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); } @@ -102,6 +108,9 @@ export async function handleSessionEnd(input) { } : null); if (sessionId && hasBrokerSessionOwners(brokerSession)) { + if (cleanupError) { + throw cleanupError; + } return; } const brokerEndpoint = brokerSession?.endpoint ?? null; @@ -123,6 +132,9 @@ export async function handleSessionEnd(input) { killProcess: terminateProcessTree }); clearBrokerSession(cwd); + if (cleanupError) { + throw cleanupError; + } } async function main() { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 2f5e42915..791359670 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -2,12 +2,14 @@ import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { spawn } from "node:child_process"; +import { pathToFileURL } from "node:url"; import test from "node:test"; import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; -import { resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; +import { resolveStateDir, resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; import { ensureBrokerSession, loadBrokerSession, @@ -114,6 +116,51 @@ async function withReadyBroker(fn) { } } +async function withHangingBroker(fn) { + const sessionDir = makeTempDir(); + const endpoint = createBrokerEndpoint(sessionDir); + const target = parseBrokerEndpoint(endpoint); + const requests = []; + const sockets = []; + const server = net.createServer((socket) => { + sockets.push(socket); + socket.setEncoding("utf8"); + socket.on("data", (chunk) => { + requests.push(chunk); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(target.path, () => { + server.off("error", reject); + resolve(); + }); + }); + + const destroySockets = () => { + for (const socket of sockets) { + socket.destroy(); + } + }; + + try { + return await fn({ endpoint, requests, sessionDir, destroySockets }); + } finally { + destroySockets(); + await new Promise((resolve) => server.close(resolve)); + } +} + +function waitForChild(child) { + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => { + resolve({ code, signal }); + }); + }); +} + test("teardownBrokersForSession tears down a broker registered for a different cwd", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); @@ -231,6 +278,138 @@ test("handleSessionEnd removes only the ending owner from a shared cwd broker", }); }); +test("concurrent SessionEnd hooks tear down a shared broker after the last owner exits", async () => { + await withPluginData(async (pluginData) => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + const markerDir = makeTempDir(); + const pidFile = path.join(sessionDir, "broker.pid"); + const logFile = path.join(sessionDir, "broker.log"); + fs.writeFileSync(pidFile, "12345\n"); + fs.writeFileSync(logFile, ""); + const brokerJson = writeBrokerJson(stateRootForTest(), "worktree-race-deadbeef", { + endpoint, + pidFile, + logFile, + sessionDir, + pid: 12345, + sessionId: "A", + sessionIds: ["A", "B"] + }); + + const moduleUrl = pathToFileURL(path.resolve("plugins/codex/scripts/lib/broker-lifecycle.mjs")).href; + const script = ` + import fs from "node:fs"; + import path from "node:path"; + + const stateFile = process.env.TEST_BROKER_STATE_FILE; + const markerDir = process.env.TEST_MARKER_DIR; + const sessionId = process.env.TEST_SESSION_ID; + const otherSessionId = sessionId === "A" ? "B" : "A"; + const originalWriteFileSync = fs.writeFileSync.bind(fs); + + fs.writeFileSync = (file, data, ...args) => { + if (file === stateFile && String(data).includes('"sessionIds"')) { + originalWriteFileSync(path.join(markerDir, sessionId + ".ready"), "", "utf8"); + const otherReady = path.join(markerDir, otherSessionId + ".ready"); + const deadline = Date.now() + 500; + while (!fs.existsSync(otherReady) && Date.now() < deadline) {} + } + return originalWriteFileSync(file, data, ...args); + }; + + const { teardownBrokersForSession } = await import(process.env.TEST_BROKER_MODULE_URL); + await teardownBrokersForSession(sessionId, { killProcess: () => {} }); + `; + + const makeChild = (sessionId) => + spawn(process.execPath, ["--input-type=module", "-e", script], { + cwd: path.resolve("."), + env: { + ...process.env, + CLAUDE_PLUGIN_DATA: pluginData, + TEST_BROKER_STATE_FILE: brokerJson, + TEST_BROKER_MODULE_URL: moduleUrl, + TEST_MARKER_DIR: markerDir, + TEST_SESSION_ID: sessionId + }, + stdio: ["ignore", "pipe", "pipe"] + }); + + const childA = makeChild("A"); + const childB = makeChild("B"); + const [resultA, resultB] = await Promise.all([waitForChild(childA), waitForChild(childB)]); + + assert.deepEqual([resultA.code, resultB.code], [0, 0]); + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(requests.length, 1); + }); + }); +}); + +test("handleSessionEnd still tears down session brokers when job cleanup fails", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const workspaceStateDir = resolveStateDir(cwd); + saveBrokerSession(cwd, { + endpoint: "unix:/tmp/codex-test-nonexistent-cleanup.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "S", + sessionIds: ["S"] + }); + const badLogFile = path.join(workspaceStateDir, "bad.log"); + fs.mkdirSync(badLogFile); + const stateFile = path.join(workspaceStateDir, "state.json"); + fs.writeFileSync( + stateFile, + `${JSON.stringify({ + version: 1, + config: { stopReviewGate: false }, + jobs: [{ id: "completed", status: "completed", sessionId: "S", logFile: badLogFile }] + }, null, 2)}\n`, + "utf8" + ); + + await assert.rejects(() => handleSessionEnd({ cwd, session_id: "S" }), { code: "EISDIR" }); + assert.equal(fs.existsSync(path.join(workspaceStateDir, "broker.json")), false); + }); +}); + +test("teardownBrokersForSession times out unresponsive broker shutdown requests", async () => { + await withPluginData(async () => { + await withHangingBroker(async ({ endpoint, requests, sessionDir, destroySockets }) => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "hanging-shutdown-deadbeef", { + endpoint, + pidFile: null, + logFile: null, + sessionDir, + pid: null, + sessionId: "S", + sessionIds: ["S"] + }); + + let completed = false; + const teardown = teardownBrokersForSession("S", { killProcess: () => {}, shutdownTimeoutMs: 50 }) + .then(() => { + completed = true; + }); + try { + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.equal(completed, true); + } finally { + destroySockets(); + await teardown; + } + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(requests.length, 1); + }); + }); +}); + test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); From a5321bd476f5f40588f3effd8d8145f6bc715787 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Tue, 7 Jul 2026 18:42:31 +0900 Subject: [PATCH 08/20] fix(broker): prune dead session owners during broker teardown A co-owner that dies without SessionEnd (SIGKILL, OOM, crash) stays in sessionIds forever, so the last graceful SessionEnd early-returns and the broker is orphaned even though every owning session is gone. Record a best-effort session pid (the SessionStart hook's parent, exported as CODEX_COMPANION_SESSION_PID) alongside each owner. teardownBrokersForSession now prunes co-owners whose recorded pid no longer exists before deciding whether the broker must stay up, and handleSessionEnd's early-return only respects owners that are still live. Owners without a recorded pid keep the previous assume-alive behavior, so legacy broker.json files are unaffected. Co-Authored-By: Claude Fable 5 --- .../codex/scripts/lib/broker-lifecycle.mjs | 83 +++++++++++-- .../codex/scripts/session-lifecycle-hook.mjs | 10 +- tests/broker-lifecycle.test.mjs | 113 +++++++++++++++++- tests/runtime.test.mjs | 4 +- 4 files changed, 193 insertions(+), 17 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 32d2e8fe5..f0dafcbc8 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -12,6 +12,7 @@ export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +export const SESSION_PID_ENV = "CODEX_COMPANION_SESSION_PID"; const BROKER_STATE_LOCK_STALE_MS = 30000; const BROKER_STATE_LOCK_TIMEOUT_MS = 5000; const BROKER_SHUTDOWN_TIMEOUT_MS = 1000; @@ -24,6 +25,15 @@ export function resolveSessionId(options = {}) { return env[SESSION_ID_ENV] ?? null; } +export function resolveSessionPid(options = {}) { + if (Number.isInteger(options.sessionPid) && options.sessionPid > 0) { + return options.sessionPid; + } + const env = options.env ?? process.env; + const parsed = Number.parseInt(env[SESSION_PID_ENV] ?? "", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + function brokerSessionOwners(session) { const owners = []; if (Array.isArray(session?.sessionIds)) { @@ -35,11 +45,55 @@ function brokerSessionOwners(session) { return [...new Set(owners.filter(Boolean))]; } -export function hasBrokerSessionOwners(session) { - return brokerSessionOwners(session).length > 0; +function brokerSessionOwnerPids(session) { + return session?.sessionPids && typeof session.sessionPids === "object" ? session.sessionPids : {}; +} + +function isProcessAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +// Owners without a recorded pid have no liveness signal and are assumed alive +// (legacy broker.json files keep their pre-pid behavior). +function isBrokerSessionOwnerLive(session, owner) { + const pid = brokerSessionOwnerPids(session)[owner]; + if (!Number.isInteger(pid) || pid <= 0) { + return true; + } + return isProcessAlive(pid); +} + +export function hasLiveBrokerSessionOwners(session) { + return brokerSessionOwners(session).some((owner) => isBrokerSessionOwnerLive(session, owner)); } -function withBrokerSessionOwner(session, sessionId) { +function brokerSessionWithOwners(session, owners) { + const pids = brokerSessionOwnerPids(session); + const keptPids = {}; + for (const owner of owners) { + if (Number.isInteger(pids[owner]) && pids[owner] > 0) { + keptPids[owner] = pids[owner]; + } + } + const next = { + ...session, + sessionId: owners[0] ?? null, + sessionIds: owners + }; + if (Object.keys(keptPids).length > 0) { + next.sessionPids = keptPids; + } else { + delete next.sessionPids; + } + return next; +} + +function withBrokerSessionOwner(session, sessionId, sessionPid = null) { if (!sessionId) { return session; } @@ -47,11 +101,11 @@ function withBrokerSessionOwner(session, sessionId) { if (!owners.includes(sessionId)) { owners.push(sessionId); } - return { - ...session, - sessionId: owners[0] ?? sessionId, - sessionIds: owners - }; + const next = brokerSessionWithOwners({ ...session }, owners); + if (Number.isInteger(sessionPid) && sessionPid > 0) { + next.sessionPids = { ...brokerSessionOwnerPids(next), [sessionId]: sessionPid }; + } + return next; } async function sleep(ms) { @@ -217,7 +271,7 @@ export async function ensureBrokerSession(cwd, options = {}) { const stateFile = resolveBrokerStateFile(cwd); return await withBrokerStateFileLock(stateFile, () => { const current = loadBrokerSession(cwd) ?? existing; - const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); + const withOwner = withBrokerSessionOwner(current, resolveSessionId(options), resolveSessionPid(options)); if (withOwner !== current) { saveBrokerSession(cwd, withOwner); } @@ -276,7 +330,7 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: child.pid ?? null, sessionId: resolveSessionId(options) }; - const withOwner = withBrokerSessionOwner(session, session.sessionId); + const withOwner = withBrokerSessionOwner(session, session.sessionId, resolveSessionPid(options)); saveBrokerSession(cwd, withOwner); return withOwner; } @@ -312,11 +366,16 @@ export async function teardownBrokersForSession(sessionId, { killProcess = null, if (!owners.includes(sessionId)) { return; } - const remainingOwners = owners.filter((owner) => owner !== sessionId); + // Drop the ending owner, then prune co-owners whose recorded session pid + // is gone — a session that died without SessionEnd must not keep the + // broker alive forever (see #380 / #108). + const remainingOwners = owners + .filter((owner) => owner !== sessionId) + .filter((owner) => isBrokerSessionOwnerLive(session, owner)); if (remainingOwners.length > 0) { fs.writeFileSync( stateFile, - `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, + `${JSON.stringify(brokerSessionWithOwners(session, remainingOwners), null, 2)}\n`, "utf8" ); return; diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index ddabc37fe..e7f70c73c 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -9,11 +9,12 @@ import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { clearBrokerSession, - hasBrokerSessionOwners, + hasLiveBrokerSessionOwners, LOG_FILE_ENV, loadBrokerSession, PID_FILE_ENV, sendBrokerShutdown, + SESSION_PID_ENV, teardownBrokerSession, teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; @@ -80,6 +81,11 @@ function cleanupSessionJobs(cwd, sessionId) { function handleSessionStart(input) { appendEnvVar(SESSION_ID_ENV, input.session_id); + // Best-effort liveness anchor for broker ownership: the hook's parent is the + // Claude process (hook commands are exec'd), which lives exactly as long as + // the session. If the capture is ever wrong, owners just look dead and broker + // teardown degrades to the pre-ownership behavior. + appendEnvVar(SESSION_PID_ENV, process.ppid); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); } @@ -107,7 +113,7 @@ export async function handleSessionEnd(input) { logFile: process.env[LOG_FILE_ENV] ?? null } : null); - if (sessionId && hasBrokerSessionOwners(brokerSession)) { + if (sessionId && hasLiveBrokerSessionOwners(brokerSession)) { if (cleanupError) { throw cleanupError; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 791359670..df842325a 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -2,7 +2,7 @@ import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { pathToFileURL } from "node:url"; import test from "node:test"; import assert from "node:assert/strict"; @@ -14,11 +14,18 @@ import { ensureBrokerSession, loadBrokerSession, resolveSessionId, + resolveSessionPid, saveBrokerSession, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; +function deadPid() { + const result = spawnSync(process.execPath, ["-e", ""]); + assert.equal(result.status, 0); + return result.pid; +} + const stateRootForTest = resolveStateRoot; test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { @@ -410,6 +417,110 @@ test("teardownBrokersForSession times out unresponsive broker shutdown requests" }); }); +test("resolveSessionPid prefers explicit option, then env, then null", () => { + assert.equal(resolveSessionPid({ sessionPid: 4321 }), 4321); + assert.equal(resolveSessionPid({ env: { CODEX_COMPANION_SESSION_PID: "1234" } }), 1234); + assert.equal(resolveSessionPid({ env: { CODEX_COMPANION_SESSION_PID: "not-a-pid" } }), null); + assert.equal(resolveSessionPid({ env: {} }), null); +}); + +test("teardownBrokersForSession prunes a dead co-owner and tears down the broker", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const stateRoot = stateRootForTest(); + const gonePid = deadPid(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-deadowner01dead", { + endpoint, + pidFile: null, + logFile: null, + sessionDir, + pid: null, + sessionId: "DEAD", + sessionIds: ["DEAD", "B"], + sessionPids: { DEAD: gonePid } + }); + + // DEAD's session pid is gone, so B is effectively the last live owner: + // its SessionEnd must shut the broker down instead of leaving it behind. + const count = await teardownBrokersForSession("B", { killProcess: () => {} }); + + assert.equal(count, 1); + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(requests.length, 1); + }); + }); +}); + +test("teardownBrokersForSession keeps a live co-owner and its recorded pid", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-liveowner1live", { + endpoint: "unix:/tmp/codex-test-nonexistent5.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "LIVE", + sessionIds: ["LIVE", "B"], + sessionPids: { LIVE: process.pid } + }); + + const count = await teardownBrokersForSession("B", { killProcess: () => {} }); + + assert.equal(count, 0); + const session = JSON.parse(fs.readFileSync(brokerJson, "utf8")); + assert.equal(session.sessionId, "LIVE"); + assert.deepEqual(session.sessionIds, ["LIVE"]); + assert.deepEqual(session.sessionPids, { LIVE: process.pid }); + }); +}); + +test("handleSessionEnd falls through to cwd teardown when the only recorded owner is dead", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const gonePid = deadPid(); + saveBrokerSession(cwd, { + endpoint: "unix:/tmp/codex-test-nonexistent6.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "DEAD", + sessionIds: ["DEAD"], + sessionPids: { DEAD: gonePid } + }); + + // The ending session never owned this broker, but its sole owner's session + // pid is gone — the legacy cwd path must reclaim it rather than early-return. + await handleSessionEnd({ cwd, session_id: "OTHER" }); + + assert.equal(loadBrokerSession(cwd), null); + }); +}); + +test("ensureBrokerSession records the reusing session's pid for liveness checks", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, + pidFile: null, + logFile: null, + sessionDir, + pid: null, + sessionId: "A" + }); + + const reused = await ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "B", CODEX_COMPANION_SESSION_PID: String(process.pid) } + }); + + assert.deepEqual(reused.sessionIds, ["A", "B"]); + assert.deepEqual(loadBrokerSession(cwd).sessionPids, { B: process.pid }); + }); + }); +}); + test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index af99274a0..843bd4861 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -694,7 +694,7 @@ test("session start hook exports the Claude session id, transcript path, and plu assert.equal(result.status, 0, result.stderr); assert.equal( fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` + `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_SESSION_PID='${process.pid}'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` ); }); @@ -726,7 +726,7 @@ test("session start hook runs when invoked through a symlinked plugin root", () assert.equal(result.status, 0, result.stderr); assert.equal( fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-symlink'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` + `export CODEX_COMPANION_SESSION_ID='sess-symlink'\nexport CODEX_COMPANION_SESSION_PID='${process.pid}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` ); }); From 19a722c63826879a5de233b160b5b9013c9ec2fe Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Tue, 7 Jul 2026 18:42:51 +0900 Subject: [PATCH 09/20] fix(hooks): guard isExecutedDirectly against unresolvable argv[1] fs.realpathSync throws ENOENT when process.argv[1] does not resolve to a real path (deleted script, loader indirection), which would crash the hook module at import time. Treat an unresolvable path as not-executed-directly. Co-Authored-By: Claude Fable 5 --- plugins/codex/scripts/session-lifecycle-hook.mjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index e7f70c73c..071ae15c7 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -162,8 +162,14 @@ function isExecutedDirectly() { return false; } - return fs.realpathSync(fileURLToPath(import.meta.url)) === - fs.realpathSync(path.resolve(process.argv[1])); + try { + return fs.realpathSync(fileURLToPath(import.meta.url)) === + fs.realpathSync(path.resolve(process.argv[1])); + } catch { + // argv[1] may not resolve to a real path (deleted file, loader indirection); + // importing must never crash at module load. + return false; + } } // Only run main() when executed directly (not when imported by tests). From da7a24beb19918f495f4950e6733a10d91a5a248 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Tue, 7 Jul 2026 18:53:19 +0900 Subject: [PATCH 10/20] fix(broker): revalidate broker state under the lock before reuse The reuse path probed readiness before acquiring the state-file lock and then fell back to the pre-lock snapshot (loadBrokerSession(cwd) ?? existing). When the last owner's SessionEnd won the lock first, the fallback resurrected the just-shut-down broker: the caller got a dead endpoint and broker.json was rewritten to point at it. Reuse now trusts only the locked re-read: if broker.json is gone or points at a different endpoint, the attempt is retried once (a live replacement broker is reused) and otherwise falls through to spawning a fresh broker. The spawn path also re-reads the current state instead of tearing down the stale pre-lock snapshot. Regression-locked with a deterministic lock-hold test (torn-down broker is not resurrected; a fresh broker is spawned) plus a live-replacement reuse test. Co-Authored-By: Claude Fable 5 --- .../codex/scripts/lib/broker-lifecycle.mjs | 37 ++++-- tests/broker-lifecycle.test.mjs | 112 ++++++++++++++++++ 2 files changed, 138 insertions(+), 11 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index f0dafcbc8..52ec4c430 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -266,26 +266,41 @@ async function isBrokerEndpointReady(endpoint) { } export async function ensureBrokerSession(cwd, options = {}) { - const existing = loadBrokerSession(cwd); - if (existing && (await isBrokerEndpointReady(existing.endpoint))) { - const stateFile = resolveBrokerStateFile(cwd); - return await withBrokerStateFileLock(stateFile, () => { - const current = loadBrokerSession(cwd) ?? existing; + const stateFile = resolveBrokerStateFile(cwd); + for (let attempt = 0; attempt < 2; attempt += 1) { + const existing = loadBrokerSession(cwd); + if (!existing || !(await isBrokerEndpointReady(existing.endpoint))) { + break; + } + // The readiness probe ran outside the lock, so the broker may have been + // torn down (or replaced) before we acquired it. Only reuse what the + // locked re-read shows; never resurrect the pre-lock snapshot. + const reused = await withBrokerStateFileLock(stateFile, () => { + const current = loadBrokerSession(cwd); + if (!current || current.endpoint !== existing.endpoint) { + return null; + } const withOwner = withBrokerSessionOwner(current, resolveSessionId(options), resolveSessionPid(options)); if (withOwner !== current) { saveBrokerSession(cwd, withOwner); } return withOwner; }); + if (reused) { + return reused; + } + // State changed while we waited for the lock — re-probe: a replacement + // broker may already be live and reusable. } - if (existing) { + const stale = loadBrokerSession(cwd); + if (stale) { teardownBrokerSession({ - endpoint: existing.endpoint ?? null, - pidFile: existing.pidFile ?? null, - logFile: existing.logFile ?? null, - sessionDir: existing.sessionDir ?? null, - pid: existing.pid ?? null, + endpoint: stale.endpoint ?? null, + pidFile: stale.pidFile ?? null, + logFile: stale.logFile ?? null, + sessionDir: stale.sessionDir ?? null, + pid: stale.pid ?? null, killProcess: options.killProcess ?? null }); clearBrokerSession(cwd); diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index df842325a..42775b256 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -26,6 +26,30 @@ function deadPid() { return result.pid; } +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// Minimal stand-in for app-server-broker.mjs: honors the spawn contract +// (serve --endpoint --cwd --pid-file) enough for waitForBrokerEndpoint. +const FAKE_BROKER_SCRIPT = `import fs from "node:fs"; +import net from "node:net"; + +const args = process.argv.slice(2); +const get = (name) => args[args.indexOf(name) + 1]; +const sockPath = get("--endpoint").replace(/^unix:/, ""); +const server = net.createServer((socket) => socket.end()); +server.listen(sockPath, () => { + fs.writeFileSync(get("--pid-file"), String(process.pid), "utf8"); +}); +`; + +function writeFakeBrokerScript() { + const scriptPath = path.join(makeTempDir(), "fake-broker.mjs"); + fs.writeFileSync(scriptPath, FAKE_BROKER_SCRIPT, "utf8"); + return scriptPath; +} + const stateRootForTest = resolveStateRoot; test("resolveStateRoot uses CLAUDE_PLUGIN_DATA/state when set", () => { @@ -521,6 +545,94 @@ test("ensureBrokerSession records the reusing session's pid for liveness checks" }); }); +test("ensureBrokerSession does not resurrect a broker torn down while waiting for the state lock", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, + pidFile: null, + logFile: null, + sessionDir, + pid: null, + sessionId: "A", + sessionIds: ["A"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + fs.mkdirSync(`${stateFile}.lock`); + + const pending = ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "B" }, + scriptPath: writeFakeBrokerScript() + }); + + // While B is parked on the lock (readiness probe already passed), A's + // SessionEnd shuts the broker down and removes broker.json. + await sleep(200); + fs.unlinkSync(stateFile); + fs.rmSync(parseBrokerEndpoint(endpoint).path, { force: true }); + fs.rmdirSync(`${stateFile}.lock`); + + const session = await pending; + try { + assert.ok(session); + assert.notEqual(session.endpoint, endpoint); + assert.deepEqual(session.sessionIds, ["B"]); + assert.equal(loadBrokerSession(cwd).endpoint, session.endpoint); + } finally { + if (session?.pid) { + try { + process.kill(session.pid); + } catch { + // Already gone. + } + } + } + }); + }); +}); + +test("ensureBrokerSession reuses a live replacement broker after losing the lock race", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint: staleEndpoint, sessionDir: staleDir }) => { + await withReadyBroker(async ({ endpoint: freshEndpoint, sessionDir: freshDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint: staleEndpoint, + pidFile: null, + logFile: null, + sessionDir: staleDir, + pid: null, + sessionId: "A", + sessionIds: ["A"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + fs.mkdirSync(`${stateFile}.lock`); + + const pending = ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "B" } }); + + // While B waits, the stale broker is replaced by a different live one. + await sleep(200); + saveBrokerSession(cwd, { + endpoint: freshEndpoint, + pidFile: null, + logFile: null, + sessionDir: freshDir, + pid: null, + sessionId: "C", + sessionIds: ["C"] + }); + fs.rmdirSync(`${stateFile}.lock`); + + const session = await pending; + assert.equal(session.endpoint, freshEndpoint); + assert.deepEqual(session.sessionIds, ["C", "B"]); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["C", "B"]); + }); + }); + }); +}); + test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); From 60671d2b816651f03890d74bc7831b9faae60691 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Tue, 7 Jul 2026 19:15:21 +0900 Subject: [PATCH 11/20] fix(broker): drop unreliable PID liveness; harden teardown/spawn concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert the session-PID owner-liveness mechanism (recording process.ppid at SessionStart and pruning owners whose pid is gone). Under the packaged shell-form SessionStart hook, node's process.ppid is the ephemeral sh that wraps the command, not the Claude session, so the recorded pid is dead almost immediately. That made every shared-broker co-owner look dead, so any single session exit tore down a broker still in use by its siblings — regressing the shared-ownership guarantee. The dead-co-owner-orphan case it targeted is the abnormal-exit concern already deferred to the broker idle-timeout (#108). Also fix two concurrency defects surfaced while reviewing that code: - teardownBrokersForSession aborted the entire state-root scan when any single broker.json.lock could not be acquired within the timeout, so a fresh lock held by an unrelated/stuck hook left the ending session's own brokers uncollected. Lock timeouts are now caught per entry and the scan continues. - ensureBrokerSession's fresh-broker spawn+persist ran outside the state-file lock, so two racing sessions could each spawn a broker and clobber broker.json, orphaning one broker with no record to clean it later. Spawn is now serialized under the lock (adopt a live broker written by a racing session, else spawn), with an unlocked best-effort fallback only when the lock cannot be acquired. - The state-file lock now stamps an owner token so a stale reclaim by another waiter cannot have its lock directory deleted by the original holder's release path. Tests: drop the pid-injection tests; add a locked-entry-skip teardown test and a dead-record fresh-spawn test. Broker suite 19/19; no new failures in the full suite (the 9 remaining failures are pre-existing environmental cases present at the merge base). Co-Authored-By: Claude Fable 5 --- .../codex/scripts/lib/broker-lifecycle.mjs | 356 ++++++++++-------- .../codex/scripts/session-lifecycle-hook.mjs | 10 +- tests/broker-lifecycle.test.mjs | 138 +++---- tests/runtime.test.mjs | 4 +- 4 files changed, 256 insertions(+), 252 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 52ec4c430..3dd89299b 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -12,10 +12,12 @@ export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; -export const SESSION_PID_ENV = "CODEX_COMPANION_SESSION_PID"; const BROKER_STATE_LOCK_STALE_MS = 30000; const BROKER_STATE_LOCK_TIMEOUT_MS = 5000; const BROKER_SHUTDOWN_TIMEOUT_MS = 1000; +const BROKER_LOCK_TIMEOUT_CODE = "EBROKERSTATELOCKTIMEOUT"; + +let brokerLockTokenSeq = 0; export function resolveSessionId(options = {}) { if (options.sessionId) { @@ -25,15 +27,6 @@ export function resolveSessionId(options = {}) { return env[SESSION_ID_ENV] ?? null; } -export function resolveSessionPid(options = {}) { - if (Number.isInteger(options.sessionPid) && options.sessionPid > 0) { - return options.sessionPid; - } - const env = options.env ?? process.env; - const parsed = Number.parseInt(env[SESSION_PID_ENV] ?? "", 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : null; -} - function brokerSessionOwners(session) { const owners = []; if (Array.isArray(session?.sessionIds)) { @@ -45,55 +38,11 @@ function brokerSessionOwners(session) { return [...new Set(owners.filter(Boolean))]; } -function brokerSessionOwnerPids(session) { - return session?.sessionPids && typeof session.sessionPids === "object" ? session.sessionPids : {}; -} - -function isProcessAlive(pid) { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return error?.code === "EPERM"; - } -} - -// Owners without a recorded pid have no liveness signal and are assumed alive -// (legacy broker.json files keep their pre-pid behavior). -function isBrokerSessionOwnerLive(session, owner) { - const pid = brokerSessionOwnerPids(session)[owner]; - if (!Number.isInteger(pid) || pid <= 0) { - return true; - } - return isProcessAlive(pid); -} - -export function hasLiveBrokerSessionOwners(session) { - return brokerSessionOwners(session).some((owner) => isBrokerSessionOwnerLive(session, owner)); -} - -function brokerSessionWithOwners(session, owners) { - const pids = brokerSessionOwnerPids(session); - const keptPids = {}; - for (const owner of owners) { - if (Number.isInteger(pids[owner]) && pids[owner] > 0) { - keptPids[owner] = pids[owner]; - } - } - const next = { - ...session, - sessionId: owners[0] ?? null, - sessionIds: owners - }; - if (Object.keys(keptPids).length > 0) { - next.sessionPids = keptPids; - } else { - delete next.sessionPids; - } - return next; +export function hasBrokerSessionOwners(session) { + return brokerSessionOwners(session).length > 0; } -function withBrokerSessionOwner(session, sessionId, sessionPid = null) { +function withBrokerSessionOwner(session, sessionId) { if (!sessionId) { return session; } @@ -101,19 +50,25 @@ function withBrokerSessionOwner(session, sessionId, sessionPid = null) { if (!owners.includes(sessionId)) { owners.push(sessionId); } - const next = brokerSessionWithOwners({ ...session }, owners); - if (Number.isInteger(sessionPid) && sessionPid > 0) { - next.sessionPids = { ...brokerSessionOwnerPids(next), [sessionId]: sessionPid }; - } - return next; + return { + ...session, + sessionId: owners[0] ?? sessionId, + sessionIds: owners + }; } async function sleep(ms) { await new Promise((resolve) => setTimeout(resolve, ms)); } +function isBrokerLockTimeout(error) { + return error?.code === BROKER_LOCK_TIMEOUT_CODE; +} + async function withBrokerStateFileLock(stateFile, fn, options = {}) { const lockDir = `${stateFile}.lock`; + const tokenFile = path.join(lockDir, "owner"); + const token = `${process.pid}-${brokerLockTokenSeq += 1}`; const timeoutMs = options.timeoutMs ?? BROKER_STATE_LOCK_TIMEOUT_MS; const staleMs = options.staleMs ?? BROKER_STATE_LOCK_STALE_MS; const deadline = Date.now() + timeoutMs; @@ -136,16 +91,34 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { continue; } if (Date.now() >= deadline) { - throw new Error(`Timed out waiting for broker state lock: ${stateFile}`); + const timeout = new Error(`Timed out waiting for broker state lock: ${stateFile}`); + timeout.code = BROKER_LOCK_TIMEOUT_CODE; + throw timeout; } await sleep(25); } } + // Stamp ownership so the finally only releases a lock we still hold — a + // stale reclaim by another waiter must not have its lock deleted from under it. + try { + fs.writeFileSync(tokenFile, token, "utf8"); + } catch { + // Non-fatal: fall back to unconditional release below. + } + try { return await fn(); } finally { - fs.rmSync(lockDir, { recursive: true, force: true }); + let owned = true; + try { + owned = fs.readFileSync(tokenFile, "utf8") === token; + } catch { + owned = false; + } + if (owned) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } } } @@ -265,47 +238,10 @@ async function isBrokerEndpointReady(endpoint) { } } -export async function ensureBrokerSession(cwd, options = {}) { - const stateFile = resolveBrokerStateFile(cwd); - for (let attempt = 0; attempt < 2; attempt += 1) { - const existing = loadBrokerSession(cwd); - if (!existing || !(await isBrokerEndpointReady(existing.endpoint))) { - break; - } - // The readiness probe ran outside the lock, so the broker may have been - // torn down (or replaced) before we acquired it. Only reuse what the - // locked re-read shows; never resurrect the pre-lock snapshot. - const reused = await withBrokerStateFileLock(stateFile, () => { - const current = loadBrokerSession(cwd); - if (!current || current.endpoint !== existing.endpoint) { - return null; - } - const withOwner = withBrokerSessionOwner(current, resolveSessionId(options), resolveSessionPid(options)); - if (withOwner !== current) { - saveBrokerSession(cwd, withOwner); - } - return withOwner; - }); - if (reused) { - return reused; - } - // State changed while we waited for the lock — re-probe: a replacement - // broker may already be live and reusable. - } - - const stale = loadBrokerSession(cwd); - if (stale) { - teardownBrokerSession({ - endpoint: stale.endpoint ?? null, - pidFile: stale.pidFile ?? null, - logFile: stale.logFile ?? null, - sessionDir: stale.sessionDir ?? null, - pid: stale.pid ?? null, - killProcess: options.killProcess ?? null - }); - clearBrokerSession(cwd); - } - +// Spawn a broker process and wait for it to accept connections. Returns an +// unsaved session record, or null if it never became ready. No locking or +// persistence — the caller owns those. +async function spawnReadyBroker(cwd, options) { const sessionDir = createBrokerSessionDir(); const endpointFactory = options.createBrokerEndpoint ?? createBrokerEndpoint; const endpoint = endpointFactory(sessionDir, options.platform); @@ -337,7 +273,7 @@ export async function ensureBrokerSession(cwd, options = {}) { return null; } - const session = { + return { endpoint, pidFile, logFile, @@ -345,12 +281,118 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: child.pid ?? null, sessionId: resolveSessionId(options) }; - const withOwner = withBrokerSessionOwner(session, session.sessionId, resolveSessionPid(options)); +} + +// Tear down whatever broker is recorded for cwd (best effort) so a fresh one +// can replace it. +function discardBrokerSession(cwd, session, options) { + if (!session) { + return; + } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess: options.killProcess ?? null + }); + clearBrokerSession(cwd); +} + +// Adopt cwd's recorded broker for this session if it is still live, else spawn +// and persist a fresh one. Callers run this inside the state-file lock so two +// racing sessions cannot each leave an orphaned broker with no broker.json. +async function adoptOrSpawnBroker(cwd, options) { + const current = loadBrokerSession(cwd); + if (current && (await isBrokerEndpointReady(current.endpoint))) { + const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); + if (withOwner !== current) { + saveBrokerSession(cwd, withOwner); + } + return withOwner; + } + discardBrokerSession(cwd, current, options); + + const session = await spawnReadyBroker(cwd, options); + if (!session) { + return null; + } + const withOwner = withBrokerSessionOwner(session, session.sessionId); saveBrokerSession(cwd, withOwner); return withOwner; } -export async function teardownBrokersForSession(sessionId, { killProcess = null, shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS } = {}) { +export async function ensureBrokerSession(cwd, options = {}) { + const stateFile = resolveBrokerStateFile(cwd); + + // Fast path: reuse a ready broker. The readiness probe runs outside the lock, + // so the broker may have been torn down (or replaced) before we acquired it; + // trust only the locked re-read, never the pre-lock snapshot. + for (let attempt = 0; attempt < 2; attempt += 1) { + const existing = loadBrokerSession(cwd); + if (!existing || !(await isBrokerEndpointReady(existing.endpoint))) { + break; + } + let lockUnavailable = false; + const reused = await withBrokerStateFileLock(stateFile, () => { + const current = loadBrokerSession(cwd); + if (!current || current.endpoint !== existing.endpoint) { + return null; + } + const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); + if (withOwner !== current) { + saveBrokerSession(cwd, withOwner); + } + return withOwner; + }).catch((error) => { + if (isBrokerLockTimeout(error)) { + lockUnavailable = true; + return null; + } + throw error; + }); + if (reused) { + return reused; + } + if (lockUnavailable) { + break; + } + // State changed while we waited for the lock — re-probe: a replacement + // broker may already be live and reusable. + } + + // Slow path: adopt-or-spawn under the lock so concurrent spawns don't orphan + // brokers. resolveStateDir must exist before we can create the lock dir. + fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); + let lockUnavailable = false; + const created = await withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options)).catch((error) => { + if (isBrokerLockTimeout(error)) { + lockUnavailable = true; + return null; + } + throw error; + }); + if (!lockUnavailable) { + return created; + } + + // Lock stayed busy past the timeout: spawn without it rather than block + // starting Codex. Rare, and degrades only to the pre-lock race window. + discardBrokerSession(cwd, loadBrokerSession(cwd), options); + const session = await spawnReadyBroker(cwd, options); + if (!session) { + return null; + } + const withOwner = withBrokerSessionOwner(session, session.sessionId); + saveBrokerSession(cwd, withOwner); + return withOwner; +} + +export async function teardownBrokersForSession( + sessionId, + { killProcess = null, shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS, lockTimeoutMs = BROKER_STATE_LOCK_TIMEOUT_MS } = {} +) { if (!sessionId) { return 0; } @@ -366,52 +408,60 @@ export async function teardownBrokersForSession(sessionId, { killProcess = null, continue; } - await withBrokerStateFileLock(stateFile, async () => { - if (!fs.existsSync(stateFile)) { - return; - } - - let session; - try { - session = JSON.parse(fs.readFileSync(stateFile, "utf8")); - } catch { - return; - } - const owners = brokerSessionOwners(session); - if (!owners.includes(sessionId)) { - return; - } - // Drop the ending owner, then prune co-owners whose recorded session pid - // is gone — a session that died without SessionEnd must not keep the - // broker alive forever (see #380 / #108). - const remainingOwners = owners - .filter((owner) => owner !== sessionId) - .filter((owner) => isBrokerSessionOwnerLive(session, owner)); - if (remainingOwners.length > 0) { - fs.writeFileSync( - stateFile, - `${JSON.stringify(brokerSessionWithOwners(session, remainingOwners), null, 2)}\n`, - "utf8" - ); - return; - } - - if (session.endpoint) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); - } - teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, - killProcess - }); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); + try { + await withBrokerStateFileLock( + stateFile, + async () => { + if (!fs.existsSync(stateFile)) { + return; + } + + let session; + try { + session = JSON.parse(fs.readFileSync(stateFile, "utf8")); + } catch { + return; + } + const owners = brokerSessionOwners(session); + if (!owners.includes(sessionId)) { + return; + } + const remainingOwners = owners.filter((owner) => owner !== sessionId); + if (remainingOwners.length > 0) { + fs.writeFileSync( + stateFile, + `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, + "utf8" + ); + return; + } + + if (session.endpoint) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); + } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + count += 1; + }, + { timeoutMs: lockTimeoutMs } + ); + } catch (error) { + if (!isBrokerLockTimeout(error)) { + throw error; } - count += 1; - }); + // Another hook holds this entry's lock and did not release it within the + // timeout. Skip it so the rest of this session's brokers still get torn + // down instead of aborting the whole scan. + } } return count; } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 071ae15c7..25b108de6 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -9,12 +9,11 @@ import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { clearBrokerSession, - hasLiveBrokerSessionOwners, + hasBrokerSessionOwners, LOG_FILE_ENV, loadBrokerSession, PID_FILE_ENV, sendBrokerShutdown, - SESSION_PID_ENV, teardownBrokerSession, teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; @@ -81,11 +80,6 @@ function cleanupSessionJobs(cwd, sessionId) { function handleSessionStart(input) { appendEnvVar(SESSION_ID_ENV, input.session_id); - // Best-effort liveness anchor for broker ownership: the hook's parent is the - // Claude process (hook commands are exec'd), which lives exactly as long as - // the session. If the capture is ever wrong, owners just look dead and broker - // teardown degrades to the pre-ownership behavior. - appendEnvVar(SESSION_PID_ENV, process.ppid); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); } @@ -113,7 +107,7 @@ export async function handleSessionEnd(input) { logFile: process.env[LOG_FILE_ENV] ?? null } : null); - if (sessionId && hasLiveBrokerSessionOwners(brokerSession)) { + if (sessionId && hasBrokerSessionOwners(brokerSession)) { if (cleanupError) { throw cleanupError; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 42775b256..d7a490a9b 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -2,7 +2,7 @@ import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { spawn, spawnSync } from "node:child_process"; +import { spawn } from "node:child_process"; import { pathToFileURL } from "node:url"; import test from "node:test"; import assert from "node:assert/strict"; @@ -14,18 +14,11 @@ import { ensureBrokerSession, loadBrokerSession, resolveSessionId, - resolveSessionPid, saveBrokerSession, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; -function deadPid() { - const result = spawnSync(process.execPath, ["-e", ""]); - assert.equal(result.status, 0); - return result.pid; -} - function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -441,107 +434,74 @@ test("teardownBrokersForSession times out unresponsive broker shutdown requests" }); }); -test("resolveSessionPid prefers explicit option, then env, then null", () => { - assert.equal(resolveSessionPid({ sessionPid: 4321 }), 4321); - assert.equal(resolveSessionPid({ env: { CODEX_COMPANION_SESSION_PID: "1234" } }), 1234); - assert.equal(resolveSessionPid({ env: { CODEX_COMPANION_SESSION_PID: "not-a-pid" } }), null); - assert.equal(resolveSessionPid({ env: {} }), null); -}); - -test("teardownBrokersForSession prunes a dead co-owner and tears down the broker", async () => { +test("teardownBrokersForSession skips a locked entry but still tears down the session's other brokers", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { const stateRoot = stateRootForTest(); - const gonePid = deadPid(); - const brokerJson = writeBrokerJson(stateRoot, "worktree-deadowner01dead", { - endpoint, - pidFile: null, - logFile: null, - sessionDir, - pid: null, - sessionId: "DEAD", - sessionIds: ["DEAD", "B"], - sessionPids: { DEAD: gonePid } - }); - // DEAD's session pid is gone, so B is effectively the last live owner: - // its SessionEnd must shut the broker down instead of leaving it behind. - const count = await teardownBrokersForSession("B", { killProcess: () => {} }); + // Entry 1: a fresh, held lock (a concurrent hook that never released it). + const lockedJson = writeBrokerJson(stateRoot, "worktree-lockedaaaaaaaa", { + endpoint: "unix:/tmp/codex-test-locked.sock", + pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" + }); + fs.mkdirSync(`${lockedJson}.lock`); - assert.equal(count, 1); - assert.equal(fs.existsSync(brokerJson), false); - assert.equal(requests.length, 1); - }); - }); -}); + // Entry 2: a real broker owned by the same session, later in the scan. + const cleanableJson = writeBrokerJson(stateRoot, "worktree-cleanablebbbb", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + }); -test("teardownBrokersForSession keeps a live co-owner and its recorded pid", async () => { - await withPluginData(async () => { - const stateRoot = stateRootForTest(); - const brokerJson = writeBrokerJson(stateRoot, "worktree-liveowner1live", { - endpoint: "unix:/tmp/codex-test-nonexistent5.sock", - pidFile: null, - logFile: null, - sessionDir: null, - pid: null, - sessionId: "LIVE", - sessionIds: ["LIVE", "B"], - sessionPids: { LIVE: process.pid } + try { + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }); + // The locked entry is skipped (not aborted), the reachable one is cleaned. + assert.equal(count, 1); + assert.equal(fs.existsSync(cleanableJson), false); + assert.equal(fs.existsSync(lockedJson), true); + assert.equal(requests.length, 1); + } finally { + fs.rmSync(`${lockedJson}.lock`, { recursive: true, force: true }); + } }); - - const count = await teardownBrokersForSession("B", { killProcess: () => {} }); - - assert.equal(count, 0); - const session = JSON.parse(fs.readFileSync(brokerJson, "utf8")); - assert.equal(session.sessionId, "LIVE"); - assert.deepEqual(session.sessionIds, ["LIVE"]); - assert.deepEqual(session.sessionPids, { LIVE: process.pid }); }); }); -test("handleSessionEnd falls through to cwd teardown when the only recorded owner is dead", async () => { +test("ensureBrokerSession spawns and persists a fresh broker when the recorded one is dead", async () => { await withPluginData(async () => { const cwd = makeTempDir(); - const gonePid = deadPid(); saveBrokerSession(cwd, { - endpoint: "unix:/tmp/codex-test-nonexistent6.sock", + endpoint: "unix:/tmp/codex-test-dead-spawn.sock", pidFile: null, logFile: null, sessionDir: null, pid: null, - sessionId: "DEAD", - sessionIds: ["DEAD"], - sessionPids: { DEAD: gonePid } + sessionId: "A", + sessionIds: ["A"] }); - // The ending session never owned this broker, but its sole owner's session - // pid is gone — the legacy cwd path must reclaim it rather than early-return. - await handleSessionEnd({ cwd, session_id: "OTHER" }); - - assert.equal(loadBrokerSession(cwd), null); - }); -}); - -test("ensureBrokerSession records the reusing session's pid for liveness checks", async () => { - await withPluginData(async () => { - await withReadyBroker(async ({ endpoint, sessionDir }) => { - const cwd = makeTempDir(); - saveBrokerSession(cwd, { - endpoint, - pidFile: null, - logFile: null, - sessionDir, - pid: null, - sessionId: "A" - }); - - const reused = await ensureBrokerSession(cwd, { - env: { CODEX_COMPANION_SESSION_ID: "B", CODEX_COMPANION_SESSION_PID: String(process.pid) } - }); - - assert.deepEqual(reused.sessionIds, ["A", "B"]); - assert.deepEqual(loadBrokerSession(cwd).sessionPids, { B: process.pid }); + const session = await ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "B" }, + scriptPath: writeFakeBrokerScript() }); + try { + assert.ok(session); + assert.notEqual(session.endpoint, "unix:/tmp/codex-test-dead-spawn.sock"); + assert.deepEqual(session.sessionIds, ["B"]); + // The freshly spawned broker is the one persisted (no orphan record). + assert.equal(loadBrokerSession(cwd).endpoint, session.endpoint); + assert.equal(await new Promise((resolve) => { + const socket = net.createConnection({ path: parseBrokerEndpoint(session.endpoint).path }); + socket.on("connect", () => { socket.end(); resolve(true); }); + socket.on("error", () => resolve(false)); + }), true); + } finally { + if (session?.pid) { + try { + process.kill(session.pid); + } catch { + // Already gone. + } + } + } }); }); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 843bd4861..af99274a0 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -694,7 +694,7 @@ test("session start hook exports the Claude session id, transcript path, and plu assert.equal(result.status, 0, result.stderr); assert.equal( fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_SESSION_PID='${process.pid}'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` + `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` ); }); @@ -726,7 +726,7 @@ test("session start hook runs when invoked through a symlinked plugin root", () assert.equal(result.status, 0, result.stderr); assert.equal( fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-symlink'\nexport CODEX_COMPANION_SESSION_PID='${process.pid}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` + `export CODEX_COMPANION_SESSION_ID='sess-symlink'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` ); }); From ee60490e14ae4764385368fdd0af1f3aacc075c1 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Wed, 8 Jul 2026 13:17:22 +0900 Subject: [PATCH 12/20] fix(broker): bound SessionEnd cleanup and don't strand the ending owner Follow-up hardening from self-review of the session-keyed broker teardown: - SessionEnd previously skipped the cwd fallback whenever broker.json still had any owner. When the session-keyed teardown was skipped under lock contention the ending session was still listed as owner, so the fallback was skipped and the broker left behind. The guard now checks for an owner OTHER than the ending session (hasOtherBrokerSessionOwners), so a broker owned only by the ended session still falls through to the cwd teardown. - teardownBrokersForSession bounded only lock acquisition, not the graceful shutdown wait, so several unresponsive endpoints could each burn the full shutdownTimeoutMs and push the scan past the SessionEnd hook budget. The shutdown wait is now capped by the remaining scan budget; out of budget skips the RPC and terminates the broker process directly. Remove the now-unused hasBrokerSessionOwners export. Add regression tests for the cwd fallthrough under lock contention and the budget-capped shutdown wait. A lock-contention variant of the dead-co-owner orphan (a skipped owner lingers in sessionIds) is a best-effort limitation backstopped by the broker idle timeout, tracked in #450. Co-Authored-By: Claude Fable 5 --- .../codex/scripts/lib/broker-lifecycle.mjs | 105 +++++++++--- .../codex/scripts/session-lifecycle-hook.mjs | 16 +- tests/broker-lifecycle.test.mjs | 152 +++++++++++++++++- 3 files changed, 247 insertions(+), 26 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 3dd89299b..1b9be3f46 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -38,8 +38,12 @@ function brokerSessionOwners(session) { return [...new Set(owners.filter(Boolean))]; } -export function hasBrokerSessionOwners(session) { - return brokerSessionOwners(session).length > 0; +// True when the broker is still owned by a session other than `sessionId`. +// Used by SessionEnd to decide whether the cwd fallback must still run: if the +// only remaining owner is the ending session (e.g. its session-keyed teardown +// was skipped under lock contention), the broker must NOT be left behind. +export function hasOtherBrokerSessionOwners(session, sessionId) { + return brokerSessionOwners(session).some((owner) => owner !== sessionId); } function withBrokerSessionOwner(session, sessionId) { @@ -84,7 +88,27 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { try { const stat = fs.statSync(lockDir); if (Date.now() - stat.mtimeMs > staleMs) { - fs.rmSync(lockDir, { recursive: true, force: true }); + // Reclaim atomically: rename the stale directory to a private name so + // only one reclaimer can take it — a blind rmSync lets two reclaimers + // each delete the other's freshly created lock. After the rename, + // re-check the mtime: if the directory we grabbed was refreshed after + // our stat (a fresh lock, not the stale one), put it back untouched. + const claimed = `${lockDir}.reclaim-${process.pid}-${(brokerLockTokenSeq += 1)}`; + try { + fs.renameSync(lockDir, claimed); + if (Date.now() - fs.statSync(claimed).mtimeMs > staleMs) { + fs.rmSync(claimed, { recursive: true, force: true }); + } else { + try { + fs.renameSync(claimed, lockDir); + } catch { + // A new lock already took the path; drop the moved copy. + fs.rmSync(claimed, { recursive: true, force: true }); + } + } + } catch { + // Lost the reclaim race; fall through and retry acquisition. + } continue; } } catch { @@ -101,20 +125,25 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { // Stamp ownership so the finally only releases a lock we still hold — a // stale reclaim by another waiter must not have its lock deleted from under it. + let stamped = false; try { fs.writeFileSync(tokenFile, token, "utf8"); + stamped = true; } catch { - // Non-fatal: fall back to unconditional release below. + // Could not stamp (quota/permission race). We still created the lock dir, + // so release it unconditionally below rather than leaking it. } try { return await fn(); } finally { let owned = true; - try { - owned = fs.readFileSync(tokenFile, "utf8") === token; - } catch { - owned = false; + if (stamped) { + try { + owned = fs.readFileSync(tokenFile, "utf8") === token; + } catch { + owned = false; + } } if (owned) { fs.rmSync(lockDir, { recursive: true, force: true }); @@ -377,21 +406,22 @@ export async function ensureBrokerSession(cwd, options = {}) { return created; } - // Lock stayed busy past the timeout: spawn without it rather than block - // starting Codex. Rare, and degrades only to the pre-lock race window. - discardBrokerSession(cwd, loadBrokerSession(cwd), options); - const session = await spawnReadyBroker(cwd, options); - if (!session) { - return null; - } - const withOwner = withBrokerSessionOwner(session, session.sessionId); - saveBrokerSession(cwd, withOwner); - return withOwner; + // Lock stayed busy past the timeout: proceed without it rather than block + // starting Codex. adopt-or-spawn still reuses a live recorded broker (and + // only discards one that fails its readiness probe), so a long-held lock + // never causes another session's live broker to be unlinked/orphaned. Rare, + // and degrades only to the pre-lock race window. + return adoptOrSpawnBroker(cwd, options); } export async function teardownBrokersForSession( sessionId, - { killProcess = null, shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS, lockTimeoutMs = BROKER_STATE_LOCK_TIMEOUT_MS } = {} + { + killProcess = null, + shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS, + lockTimeoutMs = BROKER_STATE_LOCK_TIMEOUT_MS, + budgetMs = null + } = {} ) { if (!sessionId) { return 0; @@ -401,13 +431,37 @@ export async function teardownBrokersForSession( return 0; } + const deadline = budgetMs != null ? Date.now() + budgetMs : null; let count = 0; for (const entry of fs.readdirSync(stateRoot)) { + if (deadline != null && Date.now() >= deadline) { + break; + } const stateFile = path.join(stateRoot, entry, BROKER_STATE_FILE); if (!fs.existsSync(stateFile)) { continue; } + // Ownership pre-check without the lock: never block on a lock held for a + // workspace this session does not own. The owner set only ever grows to + // include our sessionId (reuse) or shrinks when we ourselves remove it, so + // an unlocked read cannot falsely exclude a broker we own. + let preview; + try { + preview = JSON.parse(fs.readFileSync(stateFile, "utf8")); + } catch { + continue; + } + if (!brokerSessionOwners(preview).includes(sessionId)) { + continue; + } + + const remaining = deadline != null ? deadline - Date.now() : lockTimeoutMs; + if (remaining <= 0) { + break; + } + const entryLockTimeout = Math.min(lockTimeoutMs, remaining); + try { await withBrokerStateFileLock( stateFile, @@ -436,8 +490,17 @@ export async function teardownBrokersForSession( return; } + // Bound the graceful-shutdown wait by the remaining scan budget, not + // just the per-RPC default: several unresponsive endpoints could + // otherwise each burn shutdownTimeoutMs and push the whole scan past + // the SessionEnd hook's budget. Out of budget → skip the RPC and let + // teardownBrokerSession terminate the process directly. if (session.endpoint) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); + const shutdownWait = + deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; + if (shutdownWait > 0) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + } } teardownBrokerSession({ endpoint: session.endpoint ?? null, @@ -452,7 +515,7 @@ export async function teardownBrokersForSession( } count += 1; }, - { timeoutMs: lockTimeoutMs } + { timeoutMs: entryLockTimeout } ); } catch (error) { if (!isBrokerLockTimeout(error)) { diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 25b108de6..0f8bab120 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -9,7 +9,7 @@ import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { clearBrokerSession, - hasBrokerSessionOwners, + hasOtherBrokerSessionOwners, LOG_FILE_ENV, loadBrokerSession, PID_FILE_ENV, @@ -23,6 +23,12 @@ import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; +// hooks.json gives the SessionEnd hook a 5s budget. Keep the session-keyed +// broker scan well under it — a short per-entry lock wait plus a total scan +// deadline — so a single contended broker.json.lock cannot starve the rest of +// cleanup before the hook is killed. +const SESSION_END_TEARDOWN_BUDGET_MS = 3000; +const SESSION_END_LOCK_TIMEOUT_MS = 750; function readHookInput() { const raw = fs.readFileSync(0, "utf8").trim(); @@ -95,7 +101,11 @@ export async function handleSessionEnd(input) { } if (sessionId) { - await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); + await teardownBrokersForSession(sessionId, { + killProcess: terminateProcessTree, + lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS, + budgetMs: SESSION_END_TEARDOWN_BUDGET_MS + }); } const brokerSession = @@ -107,7 +117,7 @@ export async function handleSessionEnd(input) { logFile: process.env[LOG_FILE_ENV] ?? null } : null); - if (sessionId && hasBrokerSessionOwners(brokerSession)) { + if (sessionId && hasOtherBrokerSessionOwners(brokerSession, sessionId)) { if (cleanupError) { throw cleanupError; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index d7a490a9b..c7b7b10a3 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -434,12 +434,13 @@ test("teardownBrokersForSession times out unresponsive broker shutdown requests" }); }); -test("teardownBrokersForSession skips a locked entry but still tears down the session's other brokers", async () => { +test("teardownBrokersForSession skips a same-session locked entry but tears down the rest", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { const stateRoot = stateRootForTest(); - // Entry 1: a fresh, held lock (a concurrent hook that never released it). + // Entry 1: owned by this session but its lock is held by a concurrent hook + // that never released it. const lockedJson = writeBrokerJson(stateRoot, "worktree-lockedaaaaaaaa", { endpoint: "unix:/tmp/codex-test-locked.sock", pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" @@ -465,6 +466,96 @@ test("teardownBrokersForSession skips a locked entry but still tears down the se }); }); +test("teardownBrokersForSession never waits on a lock for another session's workspace", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const stateRoot = stateRootForTest(); + + // An unrelated workspace with a held lock, ordered before ours. + const otherJson = writeBrokerJson(stateRoot, "worktree-0000otheraaaa", { + endpoint: "unix:/tmp/codex-test-other.sock", + pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "OTHER" + }); + fs.mkdirSync(`${otherJson}.lock`); + + const ourJson = writeBrokerJson(stateRoot, "worktree-9999oursbbbbb", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + }); + + try { + const startedAt = Date.now(); + // A large per-entry lock timeout would stall for seconds if we waited on + // the unrelated lock; the ownership pre-check must skip it outright. + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 5000 }); + const elapsed = Date.now() - startedAt; + + assert.equal(count, 1); + assert.equal(fs.existsSync(ourJson), false); + assert.equal(fs.existsSync(otherJson), true); + assert.ok(elapsed < 1000, `expected fast teardown, took ${elapsed}ms`); + assert.equal(requests.length, 1); + } finally { + fs.rmSync(`${otherJson}.lock`, { recursive: true, force: true }); + } + }); + }); +}); + +test("teardownBrokersForSession reclaims a stale lock and still tears the broker down", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-stalelock1234", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + }); + + // A lock left behind by a crashed holder: present but aged well past the + // stale threshold, so acquisition must reclaim it instead of timing out. + const lockDir = `${brokerJson}.lock`; + fs.mkdirSync(lockDir); + const old = new Date(Date.now() - 120000); + fs.utimesSync(lockDir, old, old); + + const startedAt = Date.now(); + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 2000 }); + const elapsed = Date.now() - startedAt; + + assert.equal(count, 1); + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(requests.length, 1); + assert.ok(elapsed < 1000, `stale lock should be reclaimed promptly, took ${elapsed}ms`); + }); + }); +}); + +test("teardownBrokersForSession stops scanning once its time budget is exhausted", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + // Two same-session entries, both with held locks. With a tiny budget the + // scan must return promptly rather than spending lockTimeoutMs on each. + const a = writeBrokerJson(stateRoot, "worktree-budgetaaaaaaaa", { + endpoint: "unix:/tmp/codex-test-b1.sock", + pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" + }); + const b = writeBrokerJson(stateRoot, "worktree-budgetbbbbbbbb", { + endpoint: "unix:/tmp/codex-test-b2.sock", + pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" + }); + fs.mkdirSync(`${a}.lock`); + fs.mkdirSync(`${b}.lock`); + + try { + const startedAt = Date.now(); + await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 5000, budgetMs: 200 }); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed < 1500, `expected budget-bounded scan, took ${elapsed}ms`); + } finally { + fs.rmSync(`${a}.lock`, { recursive: true, force: true }); + fs.rmSync(`${b}.lock`, { recursive: true, force: true }); + } + }); +}); + test("ensureBrokerSession spawns and persists a fresh broker when the recorded one is dead", async () => { await withPluginData(async () => { const cwd = makeTempDir(); @@ -593,6 +684,63 @@ test("ensureBrokerSession reuses a live replacement broker after losing the lock }); }); +test("handleSessionEnd falls through to cwd teardown when session teardown is skipped under lock contention", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "S", sessionIds: ["S"] + }); + // Hold the broker's lock so the session-keyed teardown skips it; the + // record then still lists the ending session as its only owner. + const lockDir = `${path.join(resolveStateDir(cwd), "broker.json")}.lock`; + fs.mkdirSync(lockDir); + + try { + await handleSessionEnd({ cwd, session_id: "S" }); + // The cwd fallback (which does not take the lock) must still tear it + // down rather than leaving a broker owned only by the ended session. + assert.equal(loadBrokerSession(cwd), null); + assert.equal(requests.length, 1); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + }); + }); +}); + +test("teardownBrokersForSession caps shutdown waits to the remaining budget", async () => { + await withPluginData(async () => { + await withHangingBroker(async ({ endpoint, sessionDir, destroySockets }) => { + const stateRoot = stateRootForTest(); + writeBrokerJson(stateRoot, "worktree-hang1aaaaaaaaa", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "S", sessionIds: ["S"] + }); + writeBrokerJson(stateRoot, "worktree-hang2bbbbbbbbb", { + endpoint, pidFile: null, logFile: null, sessionDir: null, pid: null, + sessionId: "S", sessionIds: ["S"] + }); + + try { + const startedAt = Date.now(); + // Endpoints accept but never reply; a per-RPC 1s wait on each would blow + // the budget. The scan must honor budgetMs across shutdown waits. + await teardownBrokersForSession("S", { + killProcess: () => {}, + shutdownTimeoutMs: 1000, + budgetMs: 300 + }); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed < 900, `expected budget-capped shutdown, took ${elapsed}ms`); + } finally { + destroySockets(); + } + }); + }); +}); + test("handleSessionEnd tears down broker even when cwd mismatches (regression #380)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); From 9d56a2bde273cb555c4e2439519799e8236732c1 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Wed, 8 Jul 2026 13:44:52 +0900 Subject: [PATCH 13/20] fix(broker): best-effort shutdown on bad endpoints; atomic state writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more correctness fixes from self-review of the SessionEnd cleanup path: - A stale broker.json with a corrupt/unsupported endpoint made sendBrokerShutdown throw synchronously (parseBrokerEndpoint), which — since the teardown scan only swallows lock timeouts — aborted the entire scan and skipped the forced process/file teardown for that record and every later same-session broker. sendBrokerShutdown is now best-effort: a connect/parse failure resolves instead of rejecting, so callers always fall through to teardownBrokerSession (which already guards its own endpoint parsing). - The unlocked ownership pre-check could observe broker.json mid-write (plain writeFileSync) and treat the resulting JSON.parse failure as "not owned", skipping a broker the ending session actually owns. broker.json is now written atomically (temp + rename) so readers never see a torn file, and a pre-check parse failure falls through to the authoritative locked re-read instead of skipping the entry. Add regression tests: corrupt-endpoint teardown, unparseable broker.json in the scan, and atomic saveBrokerSession (no temp-file litter). Broker suite 27/27. The lock-contention stale-owner orphan remains a best-effort limitation backstopped by the broker idle timeout (#450). Co-Authored-By: Claude Fable 5 --- .../codex/scripts/lib/broker-lifecycle.mjs | 43 +++++++++--- tests/broker-lifecycle.test.mjs | 68 +++++++++++++++++++ 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 1b9be3f46..116157843 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -183,7 +183,6 @@ export async function sendBrokerShutdown(endpoint, { timeoutMs = BROKER_SHUTDOWN await new Promise((resolve) => { let settled = false; let timer = null; - const socket = connectToEndpoint(endpoint); const finish = () => { if (settled) { return; @@ -194,6 +193,16 @@ export async function sendBrokerShutdown(endpoint, { timeoutMs = BROKER_SHUTDOWN } resolve(); }; + // Graceful shutdown is best-effort: a corrupt/unsupported endpoint makes + // connectToEndpoint (parseBrokerEndpoint) throw synchronously. Never reject, + // so callers always fall through to the forced process/file teardown. + let socket; + try { + socket = connectToEndpoint(endpoint); + } catch { + finish(); + return; + } if (timeoutMs > 0) { timer = setTimeout(() => { socket.destroy(); @@ -243,10 +252,19 @@ export function loadBrokerSession(cwd) { } } +// Write broker.json atomically (temp + rename) so a concurrent reader — e.g. an +// unlocked ownership pre-check in another session's teardown — never observes a +// half-written file and mis-parses it. +function writeBrokerStateFile(stateFile, session) { + const tmp = `${stateFile}.tmp-${process.pid}-${(brokerLockTokenSeq += 1)}`; + fs.writeFileSync(tmp, `${JSON.stringify(session, null, 2)}\n`, "utf8"); + fs.renameSync(tmp, stateFile); +} + export function saveBrokerSession(cwd, session) { const stateDir = resolveStateDir(cwd); fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(resolveBrokerStateFile(cwd), `${JSON.stringify(session, null, 2)}\n`, "utf8"); + writeBrokerStateFile(resolveBrokerStateFile(cwd), session); } export function clearBrokerSession(cwd) { @@ -445,14 +463,17 @@ export async function teardownBrokersForSession( // Ownership pre-check without the lock: never block on a lock held for a // workspace this session does not own. The owner set only ever grows to // include our sessionId (reuse) or shrinks when we ourselves remove it, so - // an unlocked read cannot falsely exclude a broker we own. - let preview; + // an unlocked read cannot falsely exclude a broker we own. A parse failure + // (e.g. a genuinely corrupt file) is NOT treated as "not ours" — fall + // through to the locked re-read, which is authoritative, rather than + // skipping a broker that might belong to this session. + let preview = null; try { preview = JSON.parse(fs.readFileSync(stateFile, "utf8")); } catch { - continue; + preview = null; } - if (!brokerSessionOwners(preview).includes(sessionId)) { + if (preview && !brokerSessionOwners(preview).includes(sessionId)) { continue; } @@ -482,11 +503,11 @@ export async function teardownBrokersForSession( } const remainingOwners = owners.filter((owner) => owner !== sessionId); if (remainingOwners.length > 0) { - fs.writeFileSync( - stateFile, - `${JSON.stringify({ ...session, sessionId: remainingOwners[0], sessionIds: remainingOwners }, null, 2)}\n`, - "utf8" - ); + writeBrokerStateFile(stateFile, { + ...session, + sessionId: remainingOwners[0], + sessionIds: remainingOwners + }); return; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index c7b7b10a3..4b0eb8e3a 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -684,6 +684,74 @@ test("ensureBrokerSession reuses a live replacement broker after losing the lock }); }); +test("teardownBrokersForSession tolerates an unparseable broker.json and keeps cleaning later brokers", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const stateRoot = stateRootForTest(); + + // A corrupt/half-written broker.json: the unlocked ownership pre-check + // must not treat a parse failure as "not ours" and abort or skip the + // rest of the scan. + const corruptDir = path.join(stateRoot, "worktree-badjson00000"); + fs.mkdirSync(corruptDir, { recursive: true }); + fs.writeFileSync(path.join(corruptDir, "broker.json"), "{ not valid json", "utf8"); + + const goodJson = writeBrokerJson(stateRoot, "worktree-goodjson11111", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + }); + + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 200 }); + + assert.equal(count, 1); + assert.equal(fs.existsSync(goodJson), false); + assert.equal(requests.length, 1); + }); + }); +}); + +test("saveBrokerSession writes broker.json atomically", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { endpoint: "unix:/tmp/x.sock", sessionId: "S", sessionIds: ["S"] }); + const dir = resolveStateDir(cwd); + // No leftover temp files, and the persisted file parses cleanly. + const leftovers = fs.readdirSync(dir).filter((name) => name.includes("broker.json.tmp")); + assert.deepEqual(leftovers, []); + assert.equal(loadBrokerSession(cwd).sessionId, "S"); + }); +}); + +test("teardownBrokersForSession tolerates a corrupt endpoint and keeps cleaning later brokers", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const stateRoot = stateRootForTest(); + + // A stale record with an unsupported endpoint: sendBrokerShutdown would + // throw synchronously on it. It must be torn down best-effort, not abort + // the scan. + const corruptPid = 999999999; // non-existent; killProcess is mocked below + const corruptJson = writeBrokerJson(stateRoot, "worktree-corrupt00000", { + endpoint: "garbage-endpoint", pidFile: null, logFile: null, + sessionDir: null, pid: corruptPid, sessionId: "S" + }); + + // A healthy broker owned by the same session, later in the scan. + const goodJson = writeBrokerJson(stateRoot, "worktree-goodendpoint1", { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + }); + + const killed = []; + const count = await teardownBrokersForSession("S", { killProcess: (pid) => killed.push(pid) }); + + assert.equal(count, 2); + assert.equal(fs.existsSync(corruptJson), false); + assert.equal(fs.existsSync(goodJson), false); + assert.ok(killed.includes(corruptPid)); + assert.equal(requests.length, 1); + }); + }); +}); + test("handleSessionEnd falls through to cwd teardown when session teardown is skipped under lock contention", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { From ab2d491a02fc04dc4e4ce4c0fc2fceb21acff6e8 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Thu, 16 Jul 2026 21:41:04 +0900 Subject: [PATCH 14/20] fix(broker): clean cross-worktree session jobs safely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SessionEnd에서 세션 소유 작업을 전체 workspace 상태에서 먼저 정리합니다. 상태 탐색이 불완전한 경우 교차 workspace 브로커 종료를 보류합니다. 잠금 경합, stale lock, 공유 시간 예산 회귀 테스트를 보강합니다. --- .../codex/scripts/lib/broker-lifecycle.mjs | 112 +++--- .../codex/scripts/session-lifecycle-hook.mjs | 138 ++++++-- tests/broker-lifecycle.test.mjs | 319 +++++++++++++++++- 3 files changed, 482 insertions(+), 87 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 116157843..2873ae8b0 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -85,39 +85,56 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { if (error?.code !== "EEXIST") { throw error; } + let stat = null; try { - const stat = fs.statSync(lockDir); - if (Date.now() - stat.mtimeMs > staleMs) { - // Reclaim atomically: rename the stale directory to a private name so - // only one reclaimer can take it — a blind rmSync lets two reclaimers - // each delete the other's freshly created lock. After the rename, - // re-check the mtime: if the directory we grabbed was refreshed after - // our stat (a fresh lock, not the stale one), put it back untouched. - const claimed = `${lockDir}.reclaim-${process.pid}-${(brokerLockTokenSeq += 1)}`; - try { - fs.renameSync(lockDir, claimed); - if (Date.now() - fs.statSync(claimed).mtimeMs > staleMs) { + stat = fs.lstatSync(lockDir); + } catch { + // The lock may have disappeared between mkdirSync and lstatSync. Retry + // through the normal deadline/backoff path instead of spinning. + } + if (stat && !stat.isDirectory()) { + try { + // unlinkSync cannot remove a directory. If another contender replaced + // the invalid path with a real lock after lstatSync, this fails safely + // instead of renaming or deleting that contender's lock. + fs.unlinkSync(lockDir); + } catch { + // Lost the replacement race or cannot remove the invalid path. Retry + // through the normal deadline/backoff path below. + } + } + if (stat && Date.now() - stat.mtimeMs > staleMs) { + // Reclaim atomically: rename the stale directory to a private name so + // only one reclaimer can take it — a blind rmSync lets two reclaimers + // each delete the other's freshly created lock. After the rename, + // re-check the mtime: if the directory we grabbed was refreshed after + // our stat (a fresh lock, not the stale one), put it back untouched. + const claimed = `${lockDir}.reclaim-${process.pid}-${(brokerLockTokenSeq += 1)}`; + let reclaimed = false; + try { + fs.renameSync(lockDir, claimed); + if (Date.now() - fs.statSync(claimed).mtimeMs > staleMs) { + fs.rmSync(claimed, { recursive: true, force: true }); + reclaimed = true; + } else { + try { + fs.renameSync(claimed, lockDir); + } catch { + // A new lock already took the path; drop the moved copy. fs.rmSync(claimed, { recursive: true, force: true }); - } else { - try { - fs.renameSync(claimed, lockDir); - } catch { - // A new lock already took the path; drop the moved copy. - fs.rmSync(claimed, { recursive: true, force: true }); - } } - } catch { - // Lost the reclaim race; fall through and retry acquisition. } + } catch { + // Lost the reclaim race; fall through and retry acquisition. + } + if (reclaimed) { continue; } - } catch { - continue; } if (Date.now() >= deadline) { - const timeout = new Error(`Timed out waiting for broker state lock: ${stateFile}`); - timeout.code = BROKER_LOCK_TIMEOUT_CODE; - throw timeout; + throw Object.assign(new Error(`Timed out waiting for broker state lock: ${stateFile}`), { + code: BROKER_LOCK_TIMEOUT_CODE + }); } await sleep(25); } @@ -372,6 +389,7 @@ async function adoptOrSpawnBroker(cwd, options) { export async function ensureBrokerSession(cwd, options = {}) { const stateFile = resolveBrokerStateFile(cwd); + const lockOptions = options.lockTimeoutMs == null ? {} : { timeoutMs: options.lockTimeoutMs }; // Fast path: reuse a ready broker. The readiness probe runs outside the lock, // so the broker may have been torn down (or replaced) before we acquired it; @@ -381,7 +399,6 @@ export async function ensureBrokerSession(cwd, options = {}) { if (!existing || !(await isBrokerEndpointReady(existing.endpoint))) { break; } - let lockUnavailable = false; const reused = await withBrokerStateFileLock(stateFile, () => { const current = loadBrokerSession(cwd); if (!current || current.endpoint !== existing.endpoint) { @@ -392,19 +409,10 @@ export async function ensureBrokerSession(cwd, options = {}) { saveBrokerSession(cwd, withOwner); } return withOwner; - }).catch((error) => { - if (isBrokerLockTimeout(error)) { - lockUnavailable = true; - return null; - } - throw error; - }); + }, lockOptions); if (reused) { return reused; } - if (lockUnavailable) { - break; - } // State changed while we waited for the lock — re-probe: a replacement // broker may already be live and reusable. } @@ -412,24 +420,7 @@ export async function ensureBrokerSession(cwd, options = {}) { // Slow path: adopt-or-spawn under the lock so concurrent spawns don't orphan // brokers. resolveStateDir must exist before we can create the lock dir. fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); - let lockUnavailable = false; - const created = await withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options)).catch((error) => { - if (isBrokerLockTimeout(error)) { - lockUnavailable = true; - return null; - } - throw error; - }); - if (!lockUnavailable) { - return created; - } - - // Lock stayed busy past the timeout: proceed without it rather than block - // starting Codex. adopt-or-spawn still reuses a live recorded broker (and - // only discards one that fails its readiness probe), so a long-held lock - // never causes another session's live broker to be unlinked/orphaned. Rare, - // and degrades only to the pre-lock race window. - return adoptOrSpawnBroker(cwd, options); + return withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options), lockOptions); } export async function teardownBrokersForSession( @@ -451,6 +442,7 @@ export async function teardownBrokersForSession( const deadline = budgetMs != null ? Date.now() + budgetMs : null; let count = 0; + let teardownError = null; for (const entry of fs.readdirSync(stateRoot)) { if (deadline != null && Date.now() >= deadline) { break; @@ -540,13 +532,17 @@ export async function teardownBrokersForSession( ); } catch (error) { if (!isBrokerLockTimeout(error)) { - throw error; + teardownError ??= error; + continue; } - // Another hook holds this entry's lock and did not release it within the - // timeout. Skip it so the rest of this session's brokers still get torn - // down instead of aborting the whole scan. + // This entry's lock remained unavailable within the timeout. Skip it so + // the rest of this session's brokers still get torn down instead of + // aborting the whole scan. } } + if (teardownError) { + throw teardownError; + } return count; } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 0f8bab120..93a9584f6 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -17,18 +17,19 @@ import { teardownBrokerSession, teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; +import { loadState, resolveStateFile, resolveStateRoot, saveState } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; -// hooks.json gives the SessionEnd hook a 5s budget. Keep the session-keyed -// broker scan well under it — a short per-entry lock wait plus a total scan -// deadline — so a single contended broker.json.lock cannot starve the rest of -// cleanup before the hook is killed. -const SESSION_END_TEARDOWN_BUDGET_MS = 3000; +// hooks.json gives the SessionEnd hook a 5s budget. Share a shorter deadline +// across cross-workspace job discovery/cleanup and session-keyed broker +// teardown so the cwd fallback retains time before the hook is killed. +const SESSION_END_CLEANUP_BUDGET_MS = 3000; const SESSION_END_LOCK_TIMEOUT_MS = 750; +const MAX_SESSION_JOB_STATE_FILES = 1000; +const MAX_SESSION_JOB_STATE_BYTES = 1024 * 1024; function readHookInput() { const raw = fs.readFileSync(0, "utf8").trim(); @@ -49,12 +50,7 @@ function appendEnvVar(name, value) { fs.appendFileSync(process.env.CLAUDE_ENV_FILE, `export ${name}=${shellEscape(value)}\n`, "utf8"); } -function cleanupSessionJobs(cwd, sessionId) { - if (!cwd || !sessionId) { - return; - } - - const workspaceRoot = resolveWorkspaceRoot(cwd); +function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId) { const stateFile = resolveStateFile(workspaceRoot); if (!fs.existsSync(stateFile)) { return; @@ -84,6 +80,101 @@ function cleanupSessionJobs(cwd, sessionId) { }); } +function readSessionJobsFromStateFile(stateFile) { + let descriptor = null; + try { + const before = fs.lstatSync(stateFile); + if (!before.isFile() || before.size > MAX_SESSION_JOB_STATE_BYTES) { + return { jobs: [], complete: false }; + } + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + descriptor = fs.openSync(stateFile, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.size > MAX_SESSION_JOB_STATE_BYTES || + opened.dev !== before.dev || opened.ino !== before.ino) { + return { jobs: [], complete: false }; + } + const state = JSON.parse(fs.readFileSync(descriptor, "utf8")); + return { jobs: Array.isArray(state.jobs) ? state.jobs : [], complete: true }; + } catch (error) { + return { jobs: [], complete: error?.code === "ENOENT" }; + } finally { + if (descriptor != null) { + fs.closeSync(descriptor); + } + } +} + +function findSessionJobWorkspaces(cwd, sessionId, deadline) { + const workspaceRoots = new Set([resolveWorkspaceRoot(cwd)]); + const stateRoot = resolveStateRoot(); + if (!fs.existsSync(stateRoot)) { + return { workspaceRoots, complete: true }; + } + + const stateDirectory = fs.opendirSync(stateRoot); + let complete = true; + let exhausted = false; + try { + let scanned = 0; + while (scanned < MAX_SESSION_JOB_STATE_FILES && Date.now() < deadline) { + const entry = stateDirectory.readSync(); + if (!entry) { + exhausted = true; + break; + } + scanned += 1; + if (entry.isSymbolicLink()) { + complete = false; + continue; + } + if (!entry.isDirectory()) { + continue; + } + const stateFile = path.join(stateRoot, entry.name, "state.json"); + const state = readSessionJobsFromStateFile(stateFile); + complete &&= state.complete; + for (const job of state.jobs) { + if (job.sessionId === sessionId && typeof job.workspaceRoot === "string" && job.workspaceRoot) { + workspaceRoots.add(job.workspaceRoot); + } + } + } + if (!exhausted) { + complete = false; + } + } finally { + stateDirectory.closeSync(); + } + return { workspaceRoots, complete }; +} + +function cleanupSessionJobs(cwd, sessionId, deadline) { + if (!cwd || !sessionId) { + return { discoveryComplete: true, error: null }; + } + + let cleanupError = null; + let workspaceIndex = 0; + const discovery = findSessionJobWorkspaces(cwd, sessionId, deadline); + for (const workspaceRoot of discovery.workspaceRoots) { + // Always clean the hook cwd first. Bound additional workspace cleanup by + // the shared SessionEnd deadline so broker cleanup and the cwd fallback + // retain time inside the hook's 5-second limit. + if (workspaceIndex > 0 && Date.now() >= deadline) { + break; + } + workspaceIndex += 1; + try { + cleanupWorkspaceSessionJobs(workspaceRoot, sessionId); + } catch (error) { + cleanupError ??= error; + } + } + return { discoveryComplete: discovery.complete, error: cleanupError }; +} + function handleSessionStart(input) { appendEnvVar(SESSION_ID_ENV, input.session_id); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); @@ -93,19 +184,28 @@ function handleSessionStart(input) { export async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); const sessionId = input.session_id || process.env[SESSION_ID_ENV]; + const cleanupDeadline = Date.now() + SESSION_END_CLEANUP_BUDGET_MS; let cleanupError = null; + let jobDiscoveryComplete = true; try { - cleanupSessionJobs(cwd, sessionId); + const cleanup = cleanupSessionJobs(cwd, sessionId, cleanupDeadline); + cleanupError = cleanup.error; + jobDiscoveryComplete = cleanup.discoveryComplete; } catch (error) { cleanupError = error; + jobDiscoveryComplete = false; } - if (sessionId) { - await teardownBrokersForSession(sessionId, { - killProcess: terminateProcessTree, - lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS, - budgetMs: SESSION_END_TEARDOWN_BUDGET_MS - }); + if (sessionId && jobDiscoveryComplete) { + try { + await teardownBrokersForSession(sessionId, { + killProcess: terminateProcessTree, + lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS, + budgetMs: Math.max(0, cleanupDeadline - Date.now()) + }); + } catch (error) { + cleanupError ??= error; + } } const brokerSession = diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 4b0eb8e3a..201a83483 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -9,7 +9,15 @@ import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; -import { resolveStateDir, resolveStateRoot } from "../plugins/codex/scripts/lib/state.mjs"; +import { + loadState, + resolveJobFile, + resolveJobLogFile, + resolveStateDir, + resolveStateRoot, + saveState, + writeJobFile +} from "../plugins/codex/scripts/lib/state.mjs"; import { ensureBrokerSession, loadBrokerSession, @@ -28,6 +36,10 @@ function sleep(ms) { const FAKE_BROKER_SCRIPT = `import fs from "node:fs"; import net from "node:net"; +if (process.env.TEST_BROKER_SPAWN_MARKER) { + fs.writeFileSync(process.env.TEST_BROKER_SPAWN_MARKER, "spawned", "utf8"); +} + const args = process.argv.slice(2); const get = (name) => args[args.indexOf(name) + 1]; const sockPath = get("--endpoint").replace(/^unix:/, ""); @@ -330,16 +342,22 @@ test("concurrent SessionEnd hooks tear down a shared broker after the last owner const markerDir = process.env.TEST_MARKER_DIR; const sessionId = process.env.TEST_SESSION_ID; const otherSessionId = sessionId === "A" ? "B" : "A"; - const originalWriteFileSync = fs.writeFileSync.bind(fs); - - fs.writeFileSync = (file, data, ...args) => { - if (file === stateFile && String(data).includes('"sessionIds"')) { - originalWriteFileSync(path.join(markerDir, sessionId + ".ready"), "", "utf8"); + const lockDir = stateFile + ".lock"; + const originalMkdirSync = fs.mkdirSync.bind(fs); + let reachedBarrier = false; + + fs.mkdirSync = (dir, ...args) => { + if (dir === lockDir && !reachedBarrier) { + reachedBarrier = true; + fs.writeFileSync(path.join(markerDir, sessionId + ".ready"), "", "utf8"); const otherReady = path.join(markerDir, otherSessionId + ".ready"); - const deadline = Date.now() + 500; + const deadline = Date.now() + 2000; while (!fs.existsSync(otherReady) && Date.now() < deadline) {} + if (!fs.existsSync(otherReady)) { + throw new Error("other SessionEnd did not reach the lock barrier"); + } } - return originalWriteFileSync(file, data, ...args); + return originalMkdirSync(dir, ...args); }; const { teardownBrokersForSession } = await import(process.env.TEST_BROKER_MODULE_URL); @@ -402,6 +420,154 @@ test("handleSessionEnd still tears down session brokers when job cleanup fails", }); }); +test("handleSessionEnd cleans session jobs from a different --cwd workspace before broker teardown", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const jobWorkspace = makeTempDir(); + const jobId = "cross-cwd-job"; + const jobLog = resolveJobLogFile(jobWorkspace, jobId); + const job = { + id: jobId, + status: "running", + sessionId: "S", + workspaceRoot: jobWorkspace, + pid: 999999999, + logFile: jobLog + }; + fs.writeFileSync(jobLog, "running\n", "utf8"); + writeJobFile(jobWorkspace, jobId, job); + saveState(jobWorkspace, { jobs: [job] }); + saveBrokerSession(jobWorkspace, { + endpoint: "unix:/tmp/codex-test-cross-cwd.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "S", + sessionIds: ["S"] + }); + + await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + + assert.deepEqual(loadState(jobWorkspace).jobs, []); + assert.equal(fs.existsSync(resolveJobFile(jobWorkspace, jobId)), false); + assert.equal(fs.existsSync(jobLog), false); + assert.equal(loadBrokerSession(jobWorkspace), null); + }); +}); + +test("handleSessionEnd continues cross-cwd cleanup after one workspace cleanup fails", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const otherWorkspace = makeTempDir(); + const badLog = path.join(resolveStateDir(hookCwd), "bad.log"); + fs.mkdirSync(badLog, { recursive: true }); + saveState(hookCwd, { + jobs: [{ + id: "bad-job", + status: "completed", + sessionId: "S", + workspaceRoot: hookCwd, + logFile: badLog + }] + }); + + const otherJobId = "other-job"; + const otherLog = resolveJobLogFile(otherWorkspace, otherJobId); + const otherJob = { + id: otherJobId, + status: "running", + sessionId: "S", + workspaceRoot: otherWorkspace, + pid: 999999999, + logFile: otherLog + }; + fs.writeFileSync(otherLog, "running\n", "utf8"); + writeJobFile(otherWorkspace, otherJobId, otherJob); + saveState(otherWorkspace, { jobs: [otherJob] }); + saveBrokerSession(otherWorkspace, { + endpoint: "unix:/tmp/codex-test-other-workspace.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + await assert.rejects(() => handleSessionEnd({ cwd: hookCwd, session_id: "S" }), { code: "EISDIR" }); + + assert.deepEqual(loadState(otherWorkspace).jobs, []); + assert.equal(fs.existsSync(resolveJobFile(otherWorkspace, otherJobId)), false); + assert.equal(fs.existsSync(otherLog), false); + assert.equal(loadBrokerSession(otherWorkspace), null); + }); +}); + +test("handleSessionEnd preserves cross-cwd brokers when job-state discovery is incomplete", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const jobWorkspace = makeTempDir(); + const stateDir = resolveStateDir(jobWorkspace); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "state.json"), + JSON.stringify({ + jobs: [{ + id: "oversized-job", + status: "running", + sessionId: "S", + workspaceRoot: jobWorkspace, + padding: "x".repeat(1024 * 1024) + }] + }), + "utf8" + ); + saveBrokerSession(jobWorkspace, { + endpoint: "unix:/tmp/codex-test-oversized-state.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + + assert.equal(loadState(jobWorkspace).jobs.length, 1); + assert.notEqual(loadBrokerSession(jobWorkspace), null); + }); +}); + +test("handleSessionEnd gives broker teardown only the remaining shared cleanup budget", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const brokerWorkspace = makeTempDir(); + saveBrokerSession(brokerWorkspace, { + endpoint: "unix:/tmp/codex-test-shared-deadline.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + const probe = fs.opendirSync(stateRootForTest()); + const dirPrototype = Object.getPrototypeOf(probe); + probe.closeSync(); + const originalReadSync = dirPrototype.readSync; + const originalNow = Date.now; + let now = 0; + Date.now = () => now; + dirPrototype.readSync = function (...args) { + const entry = originalReadSync.call(this, ...args); + if (!entry) { + now = 3000; + } + return entry; + }; + + try { + await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + } finally { + Date.now = originalNow; + dirPrototype.readSync = originalReadSync; + } + + assert.notEqual(loadBrokerSession(brokerWorkspace), null); + }); +}); + test("teardownBrokersForSession times out unresponsive broker shutdown requests", async () => { await withPluginData(async () => { await withHangingBroker(async ({ endpoint, requests, sessionDir, destroySockets }) => { @@ -466,6 +632,57 @@ test("teardownBrokersForSession skips a same-session locked entry but tears down }); }); +test("teardownBrokersForSession replaces an invalid non-directory lock path", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-invalidlockaaa", { + endpoint: "unix:/tmp/codex-test-invalid-lock.sock", + sessionId: "S" + }); + const lockPath = `${brokerJson}.lock`; + fs.writeFileSync(lockPath, "not a directory", "utf8"); + + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }); + + assert.equal(count, 1); + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(fs.existsSync(lockPath), false); + }); +}); + +test("teardownBrokersForSession never deletes a valid lock that replaces an invalid path", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-lockswapaaaa", { + endpoint: "unix:/tmp/codex-test-lock-swap.sock", + sessionId: "S" + }); + const lockPath = `${brokerJson}.lock`; + fs.writeFileSync(lockPath, "invalid", "utf8"); + const originalLstatSync = fs.lstatSync.bind(fs); + let swapped = false; + fs.lstatSync = (file, ...args) => { + const stat = originalLstatSync(file, ...args); + if (file === lockPath && !swapped) { + swapped = true; + fs.unlinkSync(lockPath); + fs.mkdirSync(lockPath); + } + return stat; + }; + + try { + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }); + assert.equal(count, 0); + } finally { + fs.lstatSync = originalLstatSync; + fs.rmSync(lockPath, { recursive: true, force: true }); + } + + assert.equal(fs.existsSync(brokerJson), true); + }); +}); + test("teardownBrokersForSession never waits on a lock for another session's workspace", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { @@ -596,6 +813,31 @@ test("ensureBrokerSession spawns and persists a fresh broker when the recorded o }); }); +test("ensureBrokerSession never spawns without the state lock after acquisition times out", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const spawnMarker = path.join(makeTempDir(), "spawned"); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + fs.mkdirSync(`${stateFile}.lock`); + + try { + await assert.rejects( + () => ensureBrokerSession(cwd, { + scriptPath: writeFakeBrokerScript(), + lockTimeoutMs: 60, + env: { ...process.env, TEST_BROKER_SPAWN_MARKER: spawnMarker } + }), + { code: "EBROKERSTATELOCKTIMEOUT" } + ); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(fs.existsSync(spawnMarker), false); + } finally { + fs.rmSync(`${stateFile}.lock`, { recursive: true, force: true }); + } + }); +}); + test("ensureBrokerSession does not resurrect a broker torn down while waiting for the state lock", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, sessionDir }) => { @@ -712,9 +954,33 @@ test("teardownBrokersForSession tolerates an unparseable broker.json and keeps c test("saveBrokerSession writes broker.json atomically", async () => { await withPluginData(async () => { const cwd = makeTempDir(); - saveBrokerSession(cwd, { endpoint: "unix:/tmp/x.sock", sessionId: "S", sessionIds: ["S"] }); const dir = resolveStateDir(cwd); - // No leftover temp files, and the persisted file parses cleanly. + const stateFile = path.join(dir, "broker.json"); + const operations = []; + const originalWriteFileSync = fs.writeFileSync.bind(fs); + const originalRenameSync = fs.renameSync.bind(fs); + fs.writeFileSync = (file, data, ...args) => { + if (String(file).startsWith(`${stateFile}.tmp-`) || file === stateFile) { + operations.push(["write", String(file)]); + } + return originalWriteFileSync(file, data, ...args); + }; + fs.renameSync = (from, to) => { + if (to === stateFile) { + operations.push(["rename", String(from), String(to)]); + } + return originalRenameSync(from, to); + }; + try { + saveBrokerSession(cwd, { endpoint: "unix:/tmp/x.sock", sessionId: "S", sessionIds: ["S"] }); + } finally { + fs.writeFileSync = originalWriteFileSync; + fs.renameSync = originalRenameSync; + } + + assert.equal(operations[0][0], "write"); + assert.match(operations[0][1], /broker\.json\.tmp-/); + assert.deepEqual(operations[1], ["rename", operations[0][1], stateFile]); const leftovers = fs.readdirSync(dir).filter((name) => name.includes("broker.json.tmp")); assert.deepEqual(leftovers, []); assert.equal(loadBrokerSession(cwd).sessionId, "S"); @@ -752,6 +1018,39 @@ test("teardownBrokersForSession tolerates a corrupt endpoint and keeps cleaning }); }); +test("teardownBrokersForSession continues after one broker cleanup fails", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const firstJson = writeBrokerJson(stateRoot, "worktree-0000failureaaa", { + endpoint: "unix:/tmp/codex-test-cleanup-failure.sock", + sessionId: "S" + }); + const secondJson = writeBrokerJson(stateRoot, "worktree-9999successbbb", { + endpoint: "unix:/tmp/codex-test-cleanup-success.sock", + sessionId: "S" + }); + const originalUnlinkSync = fs.unlinkSync.bind(fs); + fs.unlinkSync = (file) => { + if (file === firstJson) { + throw Object.assign(new Error("simulated unlink failure"), { code: "EACCES" }); + } + return originalUnlinkSync(file); + }; + + try { + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {} }), + { code: "EACCES" } + ); + } finally { + fs.unlinkSync = originalUnlinkSync; + } + + assert.equal(fs.existsSync(firstJson), true); + assert.equal(fs.existsSync(secondJson), false); + }); +}); + test("handleSessionEnd falls through to cwd teardown when session teardown is skipped under lock contention", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { From 3cb8e208ff753f1e6b217d9e47b5a634c9da9b82 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 12:19:21 +0900 Subject: [PATCH 15/20] [COMMON] Serialize session teardown state transitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 세션 종료와 브로커 재사용이 겹쳐도 새 소유자의 브로커를 종료하지 않도록 종료 의도와 소유자 갱신을 잠금 안에서 재검증합니다. 작업 상태 갱신과 stale lock 회수를 원자화하고 교차 cwd 정리 경쟁을 회귀 테스트로 고정합니다. --- .../codex/scripts/lib/broker-lifecycle.mjs | 516 +++++++++++++++--- plugins/codex/scripts/lib/state.mjs | 271 ++++++++- .../codex/scripts/session-lifecycle-hook.mjs | 58 +- tests/broker-lifecycle.test.mjs | 184 ++++++- tests/state.test.mjs | 131 ++++- 5 files changed, 1010 insertions(+), 150 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 2873ae8b0..8083162fc 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -1,4 +1,5 @@ import fs from "node:fs"; +import { createHash } from "node:crypto"; import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -16,6 +17,8 @@ const BROKER_STATE_LOCK_STALE_MS = 30000; const BROKER_STATE_LOCK_TIMEOUT_MS = 5000; const BROKER_SHUTDOWN_TIMEOUT_MS = 1000; const BROKER_LOCK_TIMEOUT_CODE = "EBROKERSTATELOCKTIMEOUT"; +const MAX_BROKER_STATE_BYTES = 64 * 1024; +const ENDED_OWNER_MARKER_PREFIX = `${BROKER_STATE_FILE}.ended-`; let brokerLockTokenSeq = 0; @@ -38,14 +41,6 @@ function brokerSessionOwners(session) { return [...new Set(owners.filter(Boolean))]; } -// True when the broker is still owned by a session other than `sessionId`. -// Used by SessionEnd to decide whether the cwd fallback must still run: if the -// only remaining owner is the ending session (e.g. its session-keyed teardown -// was skipped under lock contention), the broker must NOT be left behind. -export function hasOtherBrokerSessionOwners(session, sessionId) { - return brokerSessionOwners(session).some((owner) => owner !== sessionId); -} - function withBrokerSessionOwner(session, sessionId) { if (!sessionId) { return session; @@ -69,6 +64,103 @@ function isBrokerLockTimeout(error) { return error?.code === BROKER_LOCK_TIMEOUT_CODE; } +function isLockOwnerAlive(token) { + const pid = Number.parseInt(token?.split("-", 1)[0] ?? "", 10); + if (!Number.isSafeInteger(pid) || pid <= 0) { + return true; + } + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } +} + +function reclaimBarrierPrefix(lockDir) { + return `${path.basename(lockDir)}.reclaim-`; +} + +function recoverBrokerReclaimBarriers(lockDir) { + const parent = path.dirname(lockDir); + const prefix = reclaimBarrierPrefix(lockDir); + let blocked = false; + for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { + if (!entry.isDirectory() || !entry.name.startsWith(prefix)) { + continue; + } + const barrier = path.join(parent, entry.name); + const reclaimerPid = Number.parseInt(entry.name.slice(prefix.length).split("-", 1)[0], 10); + if (isLockOwnerAlive(`${reclaimerPid}-reclaimer`)) { + blocked = true; + continue; + } + const movedLock = path.join(barrier, "lock"); + if (fs.existsSync(movedLock)) { + let ownerToken = null; + try { + ownerToken = fs.readFileSync(path.join(movedLock, "owner"), "utf8"); + } catch { + // An unreadable moved lock is preserved conservatively. + } + if (!ownerToken || isLockOwnerAlive(ownerToken)) { + if (!fs.existsSync(lockDir)) { + try { + fs.renameSync(movedLock, lockDir); + fs.rmSync(barrier, { recursive: true, force: true }); + continue; + } catch { + // Another process restored or published the canonical lock. + } + } + blocked = true; + continue; + } + } + fs.rmSync(barrier, { recursive: true, force: true }); + } + return blocked; +} + +function releaseOwnedBrokerLock(lockDir, token) { + const tokenFile = path.join(lockDir, "owner"); + try { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + return; + } + } catch { + // A reclaimer may have moved the lock behind a visible barrier. + } + const parent = path.dirname(lockDir); + const prefix = reclaimBarrierPrefix(lockDir); + for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { + if (!entry.isDirectory() || !entry.name.startsWith(prefix)) { + continue; + } + const barrier = path.join(parent, entry.name); + const movedLock = path.join(barrier, "lock"); + try { + if (fs.readFileSync(path.join(movedLock, "owner"), "utf8") === token) { + fs.rmSync(movedLock, { recursive: true, force: true }); + fs.rmSync(barrier, { recursive: true, force: true }); + break; + } + } catch { + // This barrier belongs to another lock generation. + } + } + // Orphan recovery can restore this generation while release is inspecting + // the moved path. Recheck the canonical token before returning. + try { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + } catch { + // This owner no longer has a published lock generation. + } +} + async function withBrokerStateFileLock(stateFile, fn, options = {}) { const lockDir = `${stateFile}.lock`; const tokenFile = path.join(lockDir, "owner"); @@ -76,13 +168,49 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { const timeoutMs = options.timeoutMs ?? BROKER_STATE_LOCK_TIMEOUT_MS; const staleMs = options.staleMs ?? BROKER_STATE_LOCK_STALE_MS; const deadline = Date.now() + timeoutMs; + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); while (true) { + if (recoverBrokerReclaimBarriers(lockDir)) { + if (Date.now() >= deadline) { + throw Object.assign(new Error(`Timed out waiting for broker state lock: ${stateFile}`), { + code: BROKER_LOCK_TIMEOUT_CODE + }); + } + await sleep(25); + continue; + } + let candidate = null; try { - fs.mkdirSync(lockDir); + if (fs.existsSync(lockDir)) { + throw Object.assign(new Error(`Broker state lock exists: ${stateFile}`), { code: "EEXIST" }); + } + candidate = fs.mkdtempSync(`${lockDir}.candidate-${process.pid}-`); + fs.writeFileSync(path.join(candidate, "owner"), token, { encoding: "utf8", flag: "wx" }); + fs.renameSync(candidate, lockDir); + candidate = null; + if (recoverBrokerReclaimBarriers(lockDir)) { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + await sleep(25); + continue; + } break; } catch (error) { - if (error?.code !== "EEXIST") { + if (candidate) { + fs.rmSync(candidate, { recursive: true, force: true }); + } + if (!fs.existsSync(lockDir)) { + if (["EEXIST", "ENOTEMPTY", "EPERM"].includes(error?.code)) { + if (Date.now() >= deadline) { + throw Object.assign(new Error(`Timed out waiting for broker state lock: ${stateFile}`), { + code: BROKER_LOCK_TIMEOUT_CODE + }); + } + await sleep(25); + continue; + } throw error; } let stat = null; @@ -93,42 +221,63 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { // through the normal deadline/backoff path instead of spinning. } if (stat && !stat.isDirectory()) { + let removed = false; try { // unlinkSync cannot remove a directory. If another contender replaced // the invalid path with a real lock after lstatSync, this fails safely // instead of renaming or deleting that contender's lock. fs.unlinkSync(lockDir); + removed = true; } catch { // Lost the replacement race or cannot remove the invalid path. Retry // through the normal deadline/backoff path below. } + if (removed) { + continue; + } + stat = null; } if (stat && Date.now() - stat.mtimeMs > staleMs) { - // Reclaim atomically: rename the stale directory to a private name so - // only one reclaimer can take it — a blind rmSync lets two reclaimers - // each delete the other's freshly created lock. After the rename, - // re-check the mtime: if the directory we grabbed was refreshed after - // our stat (a fresh lock, not the stale one), put it back untouched. - const claimed = `${lockDir}.reclaim-${process.pid}-${(brokerLockTokenSeq += 1)}`; - let reclaimed = false; + let ownerToken = null; try { - fs.renameSync(lockDir, claimed); - if (Date.now() - fs.statSync(claimed).mtimeMs > staleMs) { - fs.rmSync(claimed, { recursive: true, force: true }); - reclaimed = true; - } else { - try { - fs.renameSync(claimed, lockDir); - } catch { - // A new lock already took the path; drop the moved copy. + ownerToken = fs.readFileSync(tokenFile, "utf8"); + } catch { + // Canonical locks are atomically published with an owner token. + // An unreadable token is preserved rather than reclaimed unsafely. + } + if (ownerToken && !isLockOwnerAlive(ownerToken)) { + const barrier = fs.mkdtempSync(`${lockDir}.reclaim-${process.pid}-`); + const claimed = path.join(barrier, "lock"); + let reclaimed = false; + try { + const currentToken = fs.readFileSync(tokenFile, "utf8"); + if (currentToken !== ownerToken) { + continue; + } + fs.renameSync(lockDir, claimed); + if (fs.readFileSync(path.join(claimed, "owner"), "utf8") === ownerToken && + !isLockOwnerAlive(ownerToken)) { fs.rmSync(claimed, { recursive: true, force: true }); + reclaimed = true; + } else { + while (fs.existsSync(lockDir) && Date.now() < deadline) { + await sleep(25); + } + if (!fs.existsSync(lockDir)) { + fs.renameSync(claimed, lockDir); + } + } + } catch { + // Keep the barrier visible until the moved lock is restored or a + // later process recovers it after this reclaimer exits. + } finally { + if (!fs.existsSync(claimed)) { + fs.rmSync(barrier, { recursive: true, force: true }); } } - } catch { - // Lost the reclaim race; fall through and retry acquisition. - } - if (reclaimed) { - continue; + if (reclaimed) { + continue; + } } } if (Date.now() >= deadline) { @@ -140,31 +289,10 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { } } - // Stamp ownership so the finally only releases a lock we still hold — a - // stale reclaim by another waiter must not have its lock deleted from under it. - let stamped = false; - try { - fs.writeFileSync(tokenFile, token, "utf8"); - stamped = true; - } catch { - // Could not stamp (quota/permission race). We still created the lock dir, - // so release it unconditionally below rather than leaking it. - } - try { return await fn(); } finally { - let owned = true; - if (stamped) { - try { - owned = fs.readFileSync(tokenFile, "utf8") === token; - } catch { - owned = false; - } - } - if (owned) { - fs.rmSync(lockDir, { recursive: true, force: true }); - } + releaseOwnedBrokerLock(lockDir, token); } } @@ -197,6 +325,9 @@ export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { } export async function sendBrokerShutdown(endpoint, { timeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS } = {}) { + if (timeoutMs <= 0) { + return; + } await new Promise((resolve) => { let settled = false; let timer = null; @@ -220,12 +351,10 @@ export async function sendBrokerShutdown(endpoint, { timeoutMs = BROKER_SHUTDOWN finish(); return; } - if (timeoutMs > 0) { - timer = setTimeout(() => { - socket.destroy(); - finish(); - }, timeoutMs); - } + timer = setTimeout(() => { + socket.destroy(); + finish(); + }, timeoutMs); socket.setEncoding("utf8"); socket.on("connect", () => { socket.write(`${JSON.stringify({ id: 1, method: "broker/shutdown", params: {} })}\n`); @@ -256,16 +385,131 @@ function resolveBrokerStateFile(cwd) { return path.join(resolveStateDir(cwd), BROKER_STATE_FILE); } -export function loadBrokerSession(cwd) { - const stateFile = resolveBrokerStateFile(cwd); - if (!fs.existsSync(stateFile)) { - return null; - } +function readBoundedUtf8(descriptor, maxBytes) { + const buffer = Buffer.allocUnsafe(maxBytes + 1); + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, 0); + return bytesRead > maxBytes ? null : buffer.toString("utf8", 0, bytesRead); +} +function readBrokerStateFile(stateFile) { + let descriptor = null; try { - return JSON.parse(fs.readFileSync(stateFile, "utf8")); + const before = fs.lstatSync(stateFile); + if (!before.isFile() || before.size > MAX_BROKER_STATE_BYTES) { + return null; + } + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + descriptor = fs.openSync(stateFile, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.size > MAX_BROKER_STATE_BYTES || + opened.dev !== before.dev || opened.ino !== before.ino) { + return null; + } + const contents = readBoundedUtf8(descriptor, MAX_BROKER_STATE_BYTES); + return contents == null ? null : JSON.parse(contents); } catch { return null; + } finally { + if (descriptor != null) { + try { + fs.closeSync(descriptor); + } catch { + // Ignore a descriptor already closed by a concurrent test shim. + } + } + } +} + +export function loadBrokerSession(cwd) { + return readBrokerStateFile(resolveBrokerStateFile(cwd)); +} + +function endedOwnerMarkerFile(stateFile, sessionId) { + const digest = createHash("sha256").update(sessionId).digest("hex"); + return path.join(path.dirname(stateFile), `${ENDED_OWNER_MARKER_PREFIX}${digest}`); +} + +function recordEndedBrokerOwner(stateFile, sessionId) { + if (!sessionId || Buffer.byteLength(sessionId, "utf8") > 1024) { + return; + } + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + try { + fs.writeFileSync(endedOwnerMarkerFile(stateFile, sessionId), sessionId, { encoding: "utf8", flag: "wx" }); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + } +} + +function applyEndedBrokerOwners(stateFile, session) { + const markerFiles = []; + const endedOwners = new Set(); + for (const entry of fs.readdirSync(path.dirname(stateFile), { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.startsWith(ENDED_OWNER_MARKER_PREFIX)) { + continue; + } + const markerFile = path.join(path.dirname(stateFile), entry.name); + let descriptor = null; + try { + const before = fs.lstatSync(markerFile); + if (!before.isFile() || before.size > 1024) { + continue; + } + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + descriptor = fs.openSync(markerFile, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.size > 1024 || + opened.dev !== before.dev || opened.ino !== before.ino) { + continue; + } + const owner = readBoundedUtf8(descriptor, 1024); + const expectedName = owner + ? `${ENDED_OWNER_MARKER_PREFIX}${createHash("sha256").update(owner).digest("hex")}` + : null; + if (owner && entry.name === expectedName) { + endedOwners.add(owner); + markerFiles.push(markerFile); + } + } catch { + // Leave unreadable markers for a later cleanup attempt. + } finally { + if (descriptor != null) { + try { + fs.closeSync(descriptor); + } catch { + // Ignore a descriptor already closed by a concurrent test shim. + } + } + } + } + if (endedOwners.size === 0) { + return { session, markerFiles }; + } + if (!session) { + return { session, markerFiles }; + } + const owners = brokerSessionOwners(session).filter((owner) => !endedOwners.has(owner)); + return { + session: { + ...session, + sessionId: owners[0] ?? null, + sessionIds: owners + }, + markerFiles + }; +} + +function clearEndedOwnerMarkers(markerFiles) { + for (const markerFile of markerFiles) { + try { + fs.unlinkSync(markerFile); + } catch { + // A later locked pass can safely retry stale marker cleanup. + } } } @@ -368,15 +612,19 @@ function discardBrokerSession(cwd, session, options) { // and persist a fresh one. Callers run this inside the state-file lock so two // racing sessions cannot each leave an orphaned broker with no broker.json. async function adoptOrSpawnBroker(cwd, options) { - const current = loadBrokerSession(cwd); + const stateFile = resolveBrokerStateFile(cwd); + const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + const current = pending.session; if (current && (await isBrokerEndpointReady(current.endpoint))) { const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); - if (withOwner !== current) { + if (withOwner !== current || pending.markerFiles.length > 0) { saveBrokerSession(cwd, withOwner); } + clearEndedOwnerMarkers(pending.markerFiles); return withOwner; } discardBrokerSession(cwd, current, options); + clearEndedOwnerMarkers(pending.markerFiles); const session = await spawnReadyBroker(cwd, options); if (!session) { @@ -400,14 +648,16 @@ export async function ensureBrokerSession(cwd, options = {}) { break; } const reused = await withBrokerStateFileLock(stateFile, () => { - const current = loadBrokerSession(cwd); + const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + const current = pending.session; if (!current || current.endpoint !== existing.endpoint) { return null; } const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); - if (withOwner !== current) { + if (withOwner !== current || pending.markerFiles.length > 0) { saveBrokerSession(cwd, withOwner); } + clearEndedOwnerMarkers(pending.markerFiles); return withOwner; }, lockOptions); if (reused) { @@ -423,13 +673,89 @@ export async function ensureBrokerSession(cwd, options = {}) { return withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options), lockOptions); } +export async function teardownBrokerForCwd( + cwd, + sessionId, + { + fallbackSession = null, + killProcess = null, + shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS, + lockTimeoutMs = BROKER_STATE_LOCK_TIMEOUT_MS + } = {} +) { + const stateFile = resolveBrokerStateFile(cwd); + if (!fs.existsSync(stateFile) && !fallbackSession) { + return false; + } + fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); + // Publish the end intent before waiting for the state lock. A lock holder + // that completes while this hook is waiting can then reconcile the owner, + // and a timeout cannot leave a tombstone that arrived after the final pass. + recordEndedBrokerOwner(stateFile, sessionId); + + try { + return await withBrokerStateFileLock( + stateFile, + async () => { + const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + const current = pending.session; + const owners = brokerSessionOwners(current); + if (sessionId && owners.length > 0) { + if (!owners.includes(sessionId)) { + if (pending.markerFiles.length > 0) { + writeBrokerStateFile(stateFile, current); + clearEndedOwnerMarkers(pending.markerFiles); + } + return false; + } + const remainingOwners = owners.filter((owner) => owner !== sessionId); + if (remainingOwners.length > 0) { + writeBrokerStateFile(stateFile, { + ...current, + sessionId: remainingOwners[0], + sessionIds: remainingOwners + }); + clearEndedOwnerMarkers(pending.markerFiles); + return false; + } + } + + const session = current ?? fallbackSession; + if (session?.endpoint) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); + } + teardownBrokerSession({ + endpoint: session?.endpoint ?? null, + pidFile: session?.pidFile ?? null, + logFile: session?.logFile ?? null, + sessionDir: session?.sessionDir ?? null, + pid: session?.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + clearEndedOwnerMarkers(pending.markerFiles); + return true; + }, + { timeoutMs: lockTimeoutMs } + ); + } catch (error) { + if (isBrokerLockTimeout(error)) { + return false; + } + throw error; + } +} + export async function teardownBrokersForSession( sessionId, { killProcess = null, shutdownTimeoutMs = BROKER_SHUTDOWN_TIMEOUT_MS, lockTimeoutMs = BROKER_STATE_LOCK_TIMEOUT_MS, - budgetMs = null + budgetMs = null, + excludeCwd = null } = {} ) { if (!sessionId) { @@ -443,11 +769,18 @@ export async function teardownBrokersForSession( const deadline = budgetMs != null ? Date.now() + budgetMs : null; let count = 0; let teardownError = null; - for (const entry of fs.readdirSync(stateRoot)) { + const excludedStateFile = excludeCwd ? resolveBrokerStateFile(excludeCwd) : null; + for (const entry of fs.readdirSync(stateRoot, { withFileTypes: true })) { if (deadline != null && Date.now() >= deadline) { break; } - const stateFile = path.join(stateRoot, entry, BROKER_STATE_FILE); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + continue; + } + const stateFile = path.join(stateRoot, entry.name, BROKER_STATE_FILE); + if (stateFile === excludedStateFile) { + continue; + } if (!fs.existsSync(stateFile)) { continue; } @@ -461,13 +794,14 @@ export async function teardownBrokersForSession( // skipping a broker that might belong to this session. let preview = null; try { - preview = JSON.parse(fs.readFileSync(stateFile, "utf8")); + preview = readBrokerStateFile(stateFile); } catch { preview = null; } if (preview && !brokerSessionOwners(preview).includes(sessionId)) { continue; } + recordEndedBrokerOwner(stateFile, sessionId); const remaining = deadline != null ? deadline - Date.now() : lockTimeoutMs; if (remaining <= 0) { @@ -483,14 +817,40 @@ export async function teardownBrokersForSession( return; } - let session; - try { - session = JSON.parse(fs.readFileSync(stateFile, "utf8")); - } catch { + const pending = applyEndedBrokerOwners(stateFile, readBrokerStateFile(stateFile)); + const session = pending.session; + if (!session) { + clearEndedOwnerMarkers(pending.markerFiles); return; } const owners = brokerSessionOwners(session); if (!owners.includes(sessionId)) { + if (pending.markerFiles.length > 0) { + if (owners.length === 0) { + if (session.endpoint) { + const shutdownWait = + deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; + if (shutdownWait > 0) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + } + } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + count += 1; + } else { + writeBrokerStateFile(stateFile, session); + } + clearEndedOwnerMarkers(pending.markerFiles); + } return; } const remainingOwners = owners.filter((owner) => owner !== sessionId); @@ -500,6 +860,7 @@ export async function teardownBrokersForSession( sessionId: remainingOwners[0], sessionIds: remainingOwners }); + clearEndedOwnerMarkers(pending.markerFiles); return; } @@ -526,6 +887,7 @@ export async function teardownBrokersForSession( if (fs.existsSync(stateFile)) { fs.unlinkSync(stateFile); } + clearEndedOwnerMarkers(pending.markerFiles); count += 1; }, { timeoutMs: entryLockTimeout } diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 5f8e04b8a..baeb940ba 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -11,6 +11,11 @@ const FALLBACK_STATE_ROOT_DIR = path.join(os.tmpdir(), "codex-companion"); const STATE_FILE_NAME = "state.json"; const JOBS_DIR_NAME = "jobs"; const MAX_JOBS = 50; +const STATE_LOCK_TIMEOUT_MS = 5000; +const STATE_LOCK_STALE_MS = 30000; +const STATE_LOCK_TIMEOUT_CODE = "ESTATELOCKTIMEOUT"; + +let stateWriteSequence = 0; function nowIso() { return new Date().toISOString(); @@ -26,6 +31,234 @@ function defaultState() { }; } +function waitSynchronously(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function isLockOwnerAlive(token) { + const pid = Number.parseInt(token?.split("-", 1)[0] ?? "", 10); + if (!Number.isSafeInteger(pid) || pid <= 0) { + return true; + } + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } +} + +function reclaimBarrierPrefix(lockDir) { + return `${path.basename(lockDir)}.reclaim-`; +} + +function recoverStateReclaimBarriers(lockDir) { + const parent = path.dirname(lockDir); + const prefix = reclaimBarrierPrefix(lockDir); + let blocked = false; + for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { + if (!entry.isDirectory() || !entry.name.startsWith(prefix)) { + continue; + } + const barrier = path.join(parent, entry.name); + const reclaimerPid = Number.parseInt(entry.name.slice(prefix.length).split("-", 1)[0], 10); + if (isLockOwnerAlive(`${reclaimerPid}-reclaimer`)) { + blocked = true; + continue; + } + const movedLock = path.join(barrier, "lock"); + if (fs.existsSync(movedLock)) { + let ownerToken = null; + try { + ownerToken = fs.readFileSync(path.join(movedLock, "owner"), "utf8"); + } catch { + // An unreadable moved lock is preserved conservatively. + } + if (!ownerToken || isLockOwnerAlive(ownerToken)) { + if (!fs.existsSync(lockDir)) { + try { + fs.renameSync(movedLock, lockDir); + fs.rmSync(barrier, { recursive: true, force: true }); + continue; + } catch { + // Another process restored or published the canonical lock. + } + } + blocked = true; + continue; + } + } + fs.rmSync(barrier, { recursive: true, force: true }); + } + return blocked; +} + +function releaseOwnedStateLock(lockDir, token) { + const tokenFile = path.join(lockDir, "owner"); + try { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + return; + } + } catch { + // A reclaimer may have moved the lock behind a visible barrier. + } + const parent = path.dirname(lockDir); + const prefix = reclaimBarrierPrefix(lockDir); + for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { + if (!entry.isDirectory() || !entry.name.startsWith(prefix)) { + continue; + } + const barrier = path.join(parent, entry.name); + const movedLock = path.join(barrier, "lock"); + try { + if (fs.readFileSync(path.join(movedLock, "owner"), "utf8") === token) { + fs.rmSync(movedLock, { recursive: true, force: true }); + fs.rmSync(barrier, { recursive: true, force: true }); + break; + } + } catch { + // This barrier belongs to another lock generation. + } + } + // Orphan recovery may have restored this generation between the moved-token + // check and removal. Recheck the canonical path before returning so a + // completed owner cannot be stranded as a live-looking lock. + try { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + } catch { + // This owner no longer has a published lock generation. + } +} + +function withStateFileLock(cwd, fn, { timeoutMs = STATE_LOCK_TIMEOUT_MS } = {}) { + const stateFile = resolveStateFile(cwd); + const lockDir = `${stateFile}.lock`; + const tokenFile = path.join(lockDir, "owner"); + const token = `${process.pid}-${stateWriteSequence += 1}`; + const deadline = Date.now() + timeoutMs; + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + + while (true) { + if (recoverStateReclaimBarriers(lockDir)) { + if (Date.now() >= deadline) { + throw Object.assign(new Error(`Timed out waiting for state lock: ${stateFile}`), { + code: STATE_LOCK_TIMEOUT_CODE + }); + } + waitSynchronously(25); + continue; + } + let candidate = null; + try { + if (fs.existsSync(lockDir)) { + throw Object.assign(new Error(`State lock exists: ${stateFile}`), { code: "EEXIST" }); + } + candidate = fs.mkdtempSync(`${lockDir}.candidate-${process.pid}-`); + fs.writeFileSync(path.join(candidate, "owner"), token, { encoding: "utf8", flag: "wx" }); + // Publishing a non-empty, pre-stamped directory is atomic. A live holder + // can therefore never expose an unowned canonical lock path. + fs.renameSync(candidate, lockDir); + candidate = null; + if (recoverStateReclaimBarriers(lockDir)) { + if (fs.readFileSync(tokenFile, "utf8") === token) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + waitSynchronously(25); + continue; + } + break; + } catch (error) { + if (candidate) { + fs.rmSync(candidate, { recursive: true, force: true }); + } + if (!fs.existsSync(lockDir)) { + if (["EEXIST", "ENOTEMPTY", "EPERM"].includes(error?.code)) { + if (Date.now() >= deadline) { + throw Object.assign(new Error(`Timed out waiting for state lock: ${stateFile}`), { + code: STATE_LOCK_TIMEOUT_CODE + }); + } + waitSynchronously(25); + continue; + } + throw error; + } + let stat = null; + try { + stat = fs.lstatSync(lockDir); + } catch { + // Retry below; the lock may have disappeared after mkdirSync failed. + } + if (stat && !stat.isDirectory()) { + try { + fs.unlinkSync(lockDir); + continue; + } catch { + stat = null; + } + } + if (stat && Date.now() - stat.mtimeMs > STATE_LOCK_STALE_MS) { + let ownerToken = null; + try { + ownerToken = fs.readFileSync(tokenFile, "utf8"); + } catch { + // Every canonical lock is published with a token. Treat an unreadable + // token conservatively as live instead of risking overlapping writers. + } + if (ownerToken && !isLockOwnerAlive(ownerToken)) { + const barrier = fs.mkdtempSync(`${lockDir}.reclaim-${process.pid}-`); + const claimed = path.join(barrier, "lock"); + let reclaimed = false; + try { + const currentToken = fs.readFileSync(tokenFile, "utf8"); + if (currentToken !== ownerToken) { + continue; + } + fs.renameSync(lockDir, claimed); + if (fs.readFileSync(path.join(claimed, "owner"), "utf8") === ownerToken && + !isLockOwnerAlive(ownerToken)) { + fs.rmSync(claimed, { recursive: true, force: true }); + reclaimed = true; + } else { + while (fs.existsSync(lockDir) && Date.now() < deadline) { + waitSynchronously(25); + } + if (!fs.existsSync(lockDir)) { + fs.renameSync(claimed, lockDir); + } + } + } catch { + // The barrier stays visible until the moved lock is restored or a + // later process recovers it after this reclaimer exits. + } finally { + if (!fs.existsSync(claimed)) { + fs.rmSync(barrier, { recursive: true, force: true }); + } + } + if (reclaimed) { + continue; + } + } + } + if (Date.now() >= deadline) { + throw Object.assign(new Error(`Timed out waiting for state lock: ${stateFile}`), { + code: STATE_LOCK_TIMEOUT_CODE + }); + } + waitSynchronously(25); + } + } + + try { + return fn(); + } finally { + releaseOwnedStateLock(lockDir, token); + } +} + export function resolveStateRoot() { const pluginDataDir = process.env[PLUGIN_DATA_ENV]; return pluginDataDir ? path.join(pluginDataDir, "state") : FALLBACK_STATE_ROOT_DIR; @@ -93,7 +326,7 @@ function removeFileIfExists(filePath) { } } -export function saveState(cwd, state) { +function saveStateUnlocked(cwd, state) { const previousJobs = loadState(cwd).jobs; ensureStateDir(cwd); const nextJobs = pruneJobs(state.jobs ?? []); @@ -115,14 +348,42 @@ export function saveState(cwd, state) { removeFileIfExists(job.logFile); } - fs.writeFileSync(resolveStateFile(cwd), `${JSON.stringify(nextState, null, 2)}\n`, "utf8"); + const stateFile = resolveStateFile(cwd); + const temporaryStateFile = `${stateFile}.tmp-${process.pid}-${stateWriteSequence += 1}`; + try { + fs.writeFileSync(temporaryStateFile, `${JSON.stringify(nextState, null, 2)}\n`, "utf8"); + fs.renameSync(temporaryStateFile, stateFile); + } finally { + fs.rmSync(temporaryStateFile, { force: true }); + } return nextState; } +export function saveState(cwd, state) { + return withStateFileLock(cwd, () => saveStateUnlocked(cwd, state)); +} + export function updateState(cwd, mutate) { - const state = loadState(cwd); - mutate(state); - return saveState(cwd, state); + return withStateFileLock(cwd, () => { + const state = loadState(cwd); + mutate(state); + return saveStateUnlocked(cwd, state); + }); +} + +export function removeSessionJobs(cwd, sessionId, options = {}) { + return withStateFileLock(cwd, () => { + const state = loadState(cwd); + const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); + if (removedJobs.length === 0) { + return []; + } + saveStateUnlocked(cwd, { + ...state, + jobs: state.jobs.filter((job) => job.sessionId !== sessionId) + }); + return removedJobs; + }, options); } export function generateJobId(prefix = "job") { diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 93a9584f6..7dc4eb108 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -8,16 +8,12 @@ import { fileURLToPath } from "node:url"; import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { - clearBrokerSession, - hasOtherBrokerSessionOwners, LOG_FILE_ENV, - loadBrokerSession, PID_FILE_ENV, - sendBrokerShutdown, - teardownBrokerSession, + teardownBrokerForCwd, teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, resolveStateRoot, saveState } from "./lib/state.mjs"; +import { removeSessionJobs, resolveStateFile, resolveStateRoot } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; @@ -50,14 +46,15 @@ function appendEnvVar(name, value) { fs.appendFileSync(process.env.CLAUDE_ENV_FILE, `export ${name}=${shellEscape(value)}\n`, "utf8"); } -function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId) { +function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline) { const stateFile = resolveStateFile(workspaceRoot); if (!fs.existsSync(stateFile)) { return; } - const state = loadState(workspaceRoot); - const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); + const removedJobs = removeSessionJobs(workspaceRoot, sessionId, { + timeoutMs: Math.max(0, deadline - Date.now()) + }); if (removedJobs.length === 0) { return; } @@ -73,11 +70,6 @@ function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId) { // Ignore teardown failures during session shutdown. } } - - saveState(workspaceRoot, { - ...state, - jobs: state.jobs.filter((job) => job.sessionId !== sessionId) - }); } function readSessionJobsFromStateFile(stateFile) { @@ -167,7 +159,7 @@ function cleanupSessionJobs(cwd, sessionId, deadline) { } workspaceIndex += 1; try { - cleanupWorkspaceSessionJobs(workspaceRoot, sessionId); + cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline); } catch (error) { cleanupError ??= error; } @@ -201,47 +193,25 @@ export async function handleSessionEnd(input) { await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree, lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS, - budgetMs: Math.max(0, cleanupDeadline - Date.now()) + budgetMs: Math.max(0, cleanupDeadline - Date.now()), + excludeCwd: cwd }); } catch (error) { cleanupError ??= error; } } - const brokerSession = - loadBrokerSession(cwd) ?? - (process.env[BROKER_ENDPOINT_ENV] + await teardownBrokerForCwd(cwd, sessionId, { + fallbackSession: process.env[BROKER_ENDPOINT_ENV] ? { endpoint: process.env[BROKER_ENDPOINT_ENV], pidFile: process.env[PID_FILE_ENV] ?? null, logFile: process.env[LOG_FILE_ENV] ?? null } - : null); - if (sessionId && hasOtherBrokerSessionOwners(brokerSession, sessionId)) { - if (cleanupError) { - throw cleanupError; - } - return; - } - const brokerEndpoint = brokerSession?.endpoint ?? null; - const pidFile = brokerSession?.pidFile ?? null; - const logFile = brokerSession?.logFile ?? null; - const sessionDir = brokerSession?.sessionDir ?? null; - const pid = brokerSession?.pid ?? null; - - if (brokerEndpoint) { - await sendBrokerShutdown(brokerEndpoint); - } - - teardownBrokerSession({ - endpoint: brokerEndpoint, - pidFile, - logFile, - sessionDir, - pid, - killProcess: terminateProcessTree + : null, + killProcess: terminateProcessTree, + lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS }); - clearBrokerSession(cwd); if (cleanupError) { throw cleanupError; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 201a83483..f3a47bbd8 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -23,6 +23,8 @@ import { loadBrokerSession, resolveSessionId, saveBrokerSession, + sendBrokerShutdown, + teardownBrokerForCwd, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; @@ -31,6 +33,34 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +function observeLockAttempt(lockDir) { + const originalMkdirSync = fs.mkdirSync; + const originalExistsSync = fs.existsSync; + let notify; + const attempted = new Promise((resolve) => { + notify = resolve; + }); + fs.mkdirSync = (dir, ...args) => { + if (dir === lockDir) { + notify(); + } + return originalMkdirSync.call(fs, dir, ...args); + }; + fs.existsSync = (target) => { + if (target === lockDir) { + notify(); + } + return originalExistsSync.call(fs, target); + }; + return { + attempted, + restore() { + fs.mkdirSync = originalMkdirSync; + fs.existsSync = originalExistsSync; + } + }; +} + // Minimal stand-in for app-server-broker.mjs: honors the spawn contract // (serve --endpoint --cwd --pid-file) enough for waitForBrokerEndpoint. const FAKE_BROKER_SCRIPT = `import fs from "node:fs"; @@ -42,7 +72,7 @@ if (process.env.TEST_BROKER_SPAWN_MARKER) { const args = process.argv.slice(2); const get = (name) => args[args.indexOf(name) + 1]; -const sockPath = get("--endpoint").replace(/^unix:/, ""); +const sockPath = get("--endpoint").replace(/^(?:unix|pipe):/, ""); const server = net.createServer((socket) => socket.end()); server.listen(sockPath, () => { fs.writeFileSync(get("--pid-file"), String(process.pid), "utf8"); @@ -607,15 +637,18 @@ test("teardownBrokersForSession skips a same-session locked entry but tears down // Entry 1: owned by this session but its lock is held by a concurrent hook // that never released it. - const lockedJson = writeBrokerJson(stateRoot, "worktree-lockedaaaaaaaa", { - endpoint: "unix:/tmp/codex-test-locked.sock", - pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" + const lockedCwd = makeTempDir(); + saveBrokerSession(lockedCwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "S", sessionIds: ["S"] }); + const lockedJson = path.join(resolveStateDir(lockedCwd), "broker.json"); fs.mkdirSync(`${lockedJson}.lock`); - // Entry 2: a real broker owned by the same session, later in the scan. + // Entry 2: another record owned by the same session, later in the scan. const cleanableJson = writeBrokerJson(stateRoot, "worktree-cleanablebbbb", { - endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S" + endpoint: "invalid:endpoint", pidFile: null, logFile: null, + sessionDir: null, pid: null, sessionId: "S" }); try { @@ -624,10 +657,20 @@ test("teardownBrokersForSession skips a same-session locked entry but tears down assert.equal(count, 1); assert.equal(fs.existsSync(cleanableJson), false); assert.equal(fs.existsSync(lockedJson), true); - assert.equal(requests.length, 1); + assert.equal(requests.length, 0); } finally { fs.rmSync(`${lockedJson}.lock`, { recursive: true, force: true }); } + + // The timed-out entry keeps an end marker. A later reuse applies it under + // the lock, so B becomes the sole owner and can perform final teardown. + const reused = await ensureBrokerSession(lockedCwd, { + env: { CODEX_COMPANION_SESSION_ID: "B" } + }); + assert.deepEqual(reused.sessionIds, ["B"]); + assert.equal(await teardownBrokersForSession("B", { killProcess: () => {} }), 1); + assert.equal(fs.existsSync(lockedJson), false); + assert.equal(requests.length, 1); }); }); }); @@ -659,6 +702,8 @@ test("teardownBrokersForSession never deletes a valid lock that replaces an inva }); const lockPath = `${brokerJson}.lock`; fs.writeFileSync(lockPath, "invalid", "utf8"); + const old = new Date(Date.now() - 120000); + fs.utimesSync(lockPath, old, old); const originalLstatSync = fs.lstatSync.bind(fs); let swapped = false; fs.lstatSync = (file, ...args) => { @@ -730,6 +775,7 @@ test("teardownBrokersForSession reclaims a stale lock and still tears the broker // stale threshold, so acquisition must reclaim it instead of timing out. const lockDir = `${brokerJson}.lock`; fs.mkdirSync(lockDir); + fs.writeFileSync(path.join(lockDir, "owner"), "2147483647-crashed", "utf8"); const old = new Date(Date.now() - 120000); fs.utimesSync(lockDir, old, old); @@ -852,7 +898,9 @@ test("ensureBrokerSession does not resurrect a broker torn down while waiting fo sessionIds: ["A"] }); const stateFile = path.join(resolveStateDir(cwd), "broker.json"); - fs.mkdirSync(`${stateFile}.lock`); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + const observer = observeLockAttempt(lockDir); const pending = ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "B" }, @@ -861,10 +909,11 @@ test("ensureBrokerSession does not resurrect a broker torn down while waiting fo // While B is parked on the lock (readiness probe already passed), A's // SessionEnd shuts the broker down and removes broker.json. - await sleep(200); + await observer.attempted; + observer.restore(); fs.unlinkSync(stateFile); fs.rmSync(parseBrokerEndpoint(endpoint).path, { force: true }); - fs.rmdirSync(`${stateFile}.lock`); + fs.rmdirSync(lockDir); const session = await pending; try { @@ -900,12 +949,15 @@ test("ensureBrokerSession reuses a live replacement broker after losing the lock sessionIds: ["A"] }); const stateFile = path.join(resolveStateDir(cwd), "broker.json"); - fs.mkdirSync(`${stateFile}.lock`); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + const observer = observeLockAttempt(lockDir); const pending = ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "B" } }); // While B waits, the stale broker is replaced by a different live one. - await sleep(200); + await observer.attempted; + observer.restore(); saveBrokerSession(cwd, { endpoint: freshEndpoint, pidFile: null, @@ -915,7 +967,7 @@ test("ensureBrokerSession reuses a live replacement broker after losing the lock sessionId: "C", sessionIds: ["C"] }); - fs.rmdirSync(`${stateFile}.lock`); + fs.rmdirSync(lockDir); const session = await pending; assert.equal(session.endpoint, freshEndpoint); @@ -951,6 +1003,50 @@ test("teardownBrokersForSession tolerates an unparseable broker.json and keeps c }); }); +test("teardownBrokersForSession skips symlinked and oversized broker state", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + fs.mkdirSync(stateRoot, { recursive: true }); + const outside = makeTempDir(); + fs.writeFileSync(path.join(outside, "broker.json"), JSON.stringify({ sessionId: "S", pid: 12345 })); + fs.symlinkSync(outside, path.join(stateRoot, "symlinked-workspace")); + + const linkedFileDir = path.join(stateRoot, "linked-file-workspace"); + fs.mkdirSync(linkedFileDir); + fs.symlinkSync(path.join(outside, "broker.json"), path.join(linkedFileDir, "broker.json")); + + const oversizedDir = path.join(stateRoot, "oversized-workspace"); + fs.mkdirSync(oversizedDir); + fs.writeFileSync(path.join(oversizedDir, "broker.json"), "x".repeat(64 * 1024 + 1)); + + const killed = []; + const count = await teardownBrokersForSession("S", { killProcess: (pid) => killed.push(pid) }); + + assert.equal(count, 0); + assert.deepEqual(killed, []); + assert.equal(fs.existsSync(path.join(outside, "broker.json")), true); + }); +}); + +test("sendBrokerShutdown returns immediately for a non-positive timeout", async () => { + await withHangingBroker(async ({ endpoint, requests }) => { + await sendBrokerShutdown(endpoint, { timeoutMs: 0 }); + assert.equal(requests.length, 0); + }); +}); + +test("teardownBrokerForCwd does not create state for an unused workspace", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const stateDir = resolveStateDir(cwd); + + const tornDown = await teardownBrokerForCwd(cwd, "S"); + + assert.equal(tornDown, false); + assert.equal(fs.existsSync(stateDir), false); + }); +}); + test("saveBrokerSession writes broker.json atomically", async () => { await withPluginData(async () => { const cwd = makeTempDir(); @@ -1051,7 +1147,7 @@ test("teardownBrokersForSession continues after one broker cleanup fails", async }); }); -test("handleSessionEnd falls through to cwd teardown when session teardown is skipped under lock contention", async () => { +test("handleSessionEnd skips the cwd fallback while broker state remains locked", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { const cwd = makeTempDir(); @@ -1059,33 +1155,73 @@ test("handleSessionEnd falls through to cwd teardown when session teardown is sk endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S", sessionIds: ["S"] }); - // Hold the broker's lock so the session-keyed teardown skips it; the - // record then still lists the ending session as its only owner. + // Hold the broker's lock through both teardown attempts. The cwd fallback + // must not use its stale unlocked snapshot to tear the broker down. const lockDir = `${path.join(resolveStateDir(cwd), "broker.json")}.lock`; fs.mkdirSync(lockDir); try { await handleSessionEnd({ cwd, session_id: "S" }); - // The cwd fallback (which does not take the lock) must still tear it - // down rather than leaving a broker owned only by the ended session. - assert.equal(loadBrokerSession(cwd), null); - assert.equal(requests.length, 1); + assert.notEqual(loadBrokerSession(cwd), null); + assert.equal(requests.length, 0); } finally { fs.rmSync(lockDir, { recursive: true, force: true }); } + + const reused = await ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "B" } + }); + assert.deepEqual(reused.sessionIds, ["B"]); + await handleSessionEnd({ cwd, session_id: "B" }); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(requests.length, 1); + }); + }); +}); + +test("teardownBrokerForCwd rechecks owners under the lock before fallback teardown", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "A", sessionIds: ["A"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + const observer = observeLockAttempt(lockDir); + + const pending = teardownBrokerForCwd(cwd, "A", { + killProcess: () => {}, + lockTimeoutMs: 500 + }); + await observer.attempted; + observer.restore(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "A", sessionIds: ["A", "B"] + }); + fs.rmdirSync(lockDir); + + const tornDown = await pending; + + assert.equal(tornDown, false); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["B"]); + assert.equal(requests.length, 0); }); }); }); test("teardownBrokersForSession caps shutdown waits to the remaining budget", async () => { await withPluginData(async () => { - await withHangingBroker(async ({ endpoint, sessionDir, destroySockets }) => { + await withHangingBroker(async ({ endpoint, requests, sessionDir, destroySockets }) => { const stateRoot = stateRootForTest(); - writeBrokerJson(stateRoot, "worktree-hang1aaaaaaaaa", { + const first = writeBrokerJson(stateRoot, "worktree-hang1aaaaaaaaa", { endpoint, pidFile: null, logFile: null, sessionDir, pid: null, sessionId: "S", sessionIds: ["S"] }); - writeBrokerJson(stateRoot, "worktree-hang2bbbbbbbbb", { + const second = writeBrokerJson(stateRoot, "worktree-hang2bbbbbbbbb", { endpoint, pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S", sessionIds: ["S"] }); @@ -1101,6 +1237,8 @@ test("teardownBrokersForSession caps shutdown waits to the remaining budget", as }); const elapsed = Date.now() - startedAt; assert.ok(elapsed < 900, `expected budget-capped shutdown, took ${elapsed}ms`); + assert.equal(Number(fs.existsSync(first)) + Number(fs.existsSync(second)), 1); + assert.equal(requests.length, 1); } finally { destroySockets(); } diff --git a/tests/state.test.mjs b/tests/state.test.mjs index 0f8f57cea..756e56ec9 100644 --- a/tests/state.test.mjs +++ b/tests/state.test.mjs @@ -5,7 +5,15 @@ import test from "node:test"; import assert from "node:assert/strict"; import { makeTempDir } from "./helpers.mjs"; -import { resolveJobFile, resolveJobLogFile, resolveStateDir, resolveStateFile, saveState } from "../plugins/codex/scripts/lib/state.mjs"; +import { + loadState, + removeSessionJobs, + resolveJobFile, + resolveJobLogFile, + resolveStateDir, + resolveStateFile, + saveState +} from "../plugins/codex/scripts/lib/state.mjs"; test("resolveStateDir uses a temp-backed per-workspace directory", () => { const workspace = makeTempDir(); @@ -103,3 +111,124 @@ test("saveState prunes dropped job artifacts when indexed jobs exceed the cap", .sort() ); }); + +test("removeSessionJobs preserves a job added before acquiring the state lock", () => { + const workspace = makeTempDir(); + const stateFile = resolveStateFile(workspace); + const jobA = { id: "job-a", status: "running", sessionId: "A" }; + const jobB = { + id: "job-b", + status: "running", + sessionId: "B", + jobFile: resolveJobFile(workspace, "job-b"), + logFile: resolveJobLogFile(workspace, "job-b") + }; + saveState(workspace, { jobs: [jobA] }); + fs.writeFileSync(jobB.jobFile, JSON.stringify(jobB), "utf8"); + fs.writeFileSync(jobB.logFile, "running\n", "utf8"); + + const lockDir = `${stateFile}.lock`; + const originalRenameSync = fs.renameSync; + let injected = false; + fs.renameSync = (source, destination) => { + if (destination === lockDir && !injected) { + injected = true; + const state = JSON.parse(fs.readFileSync(stateFile, "utf8")); + fs.writeFileSync(stateFile, `${JSON.stringify({ ...state, jobs: [...state.jobs, jobB] }, null, 2)}\n`, "utf8"); + } + return originalRenameSync.call(fs, source, destination); + }; + + let removed; + try { + removed = removeSessionJobs(workspace, "A"); + } finally { + fs.renameSync = originalRenameSync; + } + + assert.deepEqual(removed.map((job) => job.id), ["job-a"]); + assert.deepEqual(loadState(workspace).jobs.map((job) => job.id), ["job-b"]); + assert.equal(fs.existsSync(jobB.jobFile), true); + assert.equal(fs.existsSync(jobB.logFile), true); +}); + +test("saveState does not release a state lock that has a replacement owner", () => { + const workspace = makeTempDir(); + const stateFile = resolveStateFile(workspace); + const lockDir = `${stateFile}.lock`; + const tokenFile = path.join(lockDir, "owner"); + const originalRenameSync = fs.renameSync; + let replaced = false; + fs.renameSync = (source, destination) => { + const result = originalRenameSync.call(fs, source, destination); + if (destination === stateFile && !replaced) { + replaced = true; + fs.writeFileSync(tokenFile, "replacement-owner", "utf8"); + } + return result; + }; + + try { + saveState(workspace, { jobs: [] }); + } finally { + fs.renameSync = originalRenameSync; + } + + assert.equal(replaced, true); + assert.equal(fs.readFileSync(tokenFile, "utf8"), "replacement-owner"); + fs.rmSync(lockDir, { recursive: true, force: true }); +}); + +test("saveState reclaims an aged lock whose owner process exited", () => { + const workspace = makeTempDir(); + const stateFile = resolveStateFile(workspace); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir, { recursive: true }); + fs.writeFileSync(path.join(lockDir, "owner"), "2147483647-crashed", "utf8"); + const old = new Date(Date.now() - 120000); + fs.utimesSync(lockDir, old, old); + + saveState(workspace, { jobs: [] }); + + assert.equal(fs.existsSync(lockDir), false); + assert.deepEqual(loadState(workspace).jobs, []); +}); + +test("saveState retries when a contended lock disappears before inspection", () => { + const workspace = makeTempDir(); + const stateFile = resolveStateFile(workspace); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir, { recursive: true }); + fs.writeFileSync(path.join(lockDir, "owner"), `${process.pid}-holder`, "utf8"); + const originalExistsSync = fs.existsSync; + let released = false; + fs.existsSync = (target) => { + if (target === lockDir && !released) { + released = true; + fs.rmSync(lockDir, { recursive: true, force: true }); + return true; + } + return originalExistsSync.call(fs, target); + }; + + try { + saveState(workspace, { jobs: [] }); + } finally { + fs.existsSync = originalExistsSync; + } + + assert.equal(released, true); + assert.deepEqual(loadState(workspace).jobs, []); +}); + +test("saveState recovers an orphaned stale-reclaim barrier", () => { + const workspace = makeTempDir(); + const stateFile = resolveStateFile(workspace); + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + const barrier = fs.mkdtempSync(`${stateFile}.lock.reclaim-2147483647-`); + + saveState(workspace, { jobs: [] }); + + assert.equal(fs.existsSync(barrier), false); + assert.deepEqual(loadState(workspace).jobs, []); +}); From 45d45e81cf2ceaa148329f87c6f36955e5440f53 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 13:10:03 +0900 Subject: [PATCH 16/20] [COMMON] Terminate session jobs before cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SessionEnd 정리 순서 회귀 수정: - state lock 안에서 세션 job 종료 신호를 먼저 전송 - artifact 삭제 실패 시 job state를 보존해 재시도 가능하게 유지 - 삭제 오류에도 broker teardown이 계속되는 회귀 테스트 보강 --- plugins/codex/scripts/lib/state.mjs | 4 ++- .../codex/scripts/session-lifecycle-hook.mjs | 36 ++++++++++--------- tests/broker-lifecycle.test.mjs | 18 ++++++++-- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index baeb940ba..ba318f10f 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -372,18 +372,20 @@ export function updateState(cwd, mutate) { } export function removeSessionJobs(cwd, sessionId, options = {}) { + const { beforeRemove = () => {}, ...lockOptions } = options; return withStateFileLock(cwd, () => { const state = loadState(cwd); const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); if (removedJobs.length === 0) { return []; } + beforeRemove(removedJobs); saveStateUnlocked(cwd, { ...state, jobs: state.jobs.filter((job) => job.sessionId !== sessionId) }); return removedJobs; - }, options); + }, lockOptions); } export function generateJobId(prefix = "job") { diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 7dc4eb108..6101ee5ed 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -52,24 +52,26 @@ function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline) { return; } - const removedJobs = removeSessionJobs(workspaceRoot, sessionId, { - timeoutMs: Math.max(0, deadline - Date.now()) - }); - if (removedJobs.length === 0) { - return; - } - - for (const job of removedJobs) { - const stillRunning = job.status === "queued" || job.status === "running"; - if (!stillRunning) { - continue; - } - try { - terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. + removeSessionJobs( + workspaceRoot, + sessionId, + { + timeoutMs: Math.max(0, deadline - Date.now()), + beforeRemove(jobs) { + for (const job of jobs) { + const stillRunning = job.status === "queued" || job.status === "running"; + if (!stillRunning) { + continue; + } + try { + terminateProcessTree(job.pid ?? Number.NaN); + } catch { + // Ignore teardown failures during session shutdown. + } + } + } } - } + ); } function readSessionJobsFromStateFile(stateFile) { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index f3a47bbd8..dca66d9de 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -434,18 +434,32 @@ test("handleSessionEnd still tears down session brokers when job cleanup fails", }); const badLogFile = path.join(workspaceStateDir, "bad.log"); fs.mkdirSync(badLogFile); + const originalKill = process.kill; + const killedPids = []; + process.kill = (pid, signal) => { + assert.deepEqual(loadState(cwd).jobs.map((job) => job.id), ["running"]); + assert.equal(fs.existsSync(badLogFile), true); + killedPids.push({ pid, signal }); + return true; + }; const stateFile = path.join(workspaceStateDir, "state.json"); fs.writeFileSync( stateFile, `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, - jobs: [{ id: "completed", status: "completed", sessionId: "S", logFile: badLogFile }] + jobs: [{ id: "running", status: "running", sessionId: "S", pid: 12345, logFile: badLogFile }] }, null, 2)}\n`, "utf8" ); - await assert.rejects(() => handleSessionEnd({ cwd, session_id: "S" }), { code: "EISDIR" }); + try { + await assert.rejects(() => handleSessionEnd({ cwd, session_id: "S" }), { code: "EISDIR" }); + } finally { + process.kill = originalKill; + } + assert.deepEqual(killedPids, [{ pid: -12345, signal: "SIGTERM" }]); + assert.deepEqual(loadState(cwd).jobs.map((job) => job.id), ["running"]); assert.equal(fs.existsSync(path.join(workspaceStateDir, "broker.json")), false); }); }); From e74ed592c14fc5d950d961d4db614e00739a891d Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 13:28:20 +0900 Subject: [PATCH 17/20] [COMMON] Preserve brokers before job cleanup starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SessionEnd 정리 안전성 및 테스트 정돈: - job state를 읽거나 lock 할 수 없으면 broker teardown 보류 - 종료된 owner의 fresh state lock을 즉시 안전하게 회수 - 더 강한 lock 경계 테스트에 포함된 중복 테스트 삭제 --- plugins/codex/scripts/lib/state.mjs | 3 +- .../codex/scripts/session-lifecycle-hook.mjs | 96 ++++++++++++------- tests/broker-lifecycle.test.mjs | 86 ++++++++++++++--- tests/state.test.mjs | 6 +- 4 files changed, 141 insertions(+), 50 deletions(-) diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index ba318f10f..ac260859f 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -12,7 +12,6 @@ const STATE_FILE_NAME = "state.json"; const JOBS_DIR_NAME = "jobs"; const MAX_JOBS = 50; const STATE_LOCK_TIMEOUT_MS = 5000; -const STATE_LOCK_STALE_MS = 30000; const STATE_LOCK_TIMEOUT_CODE = "ESTATELOCKTIMEOUT"; let stateWriteSequence = 0; @@ -200,7 +199,7 @@ function withStateFileLock(cwd, fn, { timeoutMs = STATE_LOCK_TIMEOUT_MS } = {}) stat = null; } } - if (stat && Date.now() - stat.mtimeMs > STATE_LOCK_STALE_MS) { + if (stat) { let ownerToken = null; try { ownerToken = fs.readFileSync(tokenFile, "utf8"); diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 6101ee5ed..5cbec7a3c 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -49,29 +49,39 @@ function appendEnvVar(name, value) { function cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline) { const stateFile = resolveStateFile(workspaceRoot); if (!fs.existsSync(stateFile)) { - return; + return { brokerTeardownSafe: true, error: null }; + } + if (!readSessionJobsFromStateFile(stateFile).complete) { + return { brokerTeardownSafe: false, error: null }; } - removeSessionJobs( - workspaceRoot, - sessionId, - { - timeoutMs: Math.max(0, deadline - Date.now()), - beforeRemove(jobs) { - for (const job of jobs) { - const stillRunning = job.status === "queued" || job.status === "running"; - if (!stillRunning) { - continue; - } - try { - terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. + let reachedTerminationPhase = false; + try { + removeSessionJobs( + workspaceRoot, + sessionId, + { + timeoutMs: Math.max(0, deadline - Date.now()), + beforeRemove(jobs) { + reachedTerminationPhase = true; + for (const job of jobs) { + const stillRunning = job.status === "queued" || job.status === "running"; + if (!stillRunning) { + continue; + } + try { + terminateProcessTree(job.pid ?? Number.NaN); + } catch { + // Ignore teardown failures during session shutdown. + } } } } - } - ); + ); + return { brokerTeardownSafe: true, error: null }; + } catch (error) { + return { brokerTeardownSafe: reachedTerminationPhase, error }; + } } function readSessionJobsFromStateFile(stateFile) { @@ -146,10 +156,12 @@ function findSessionJobWorkspaces(cwd, sessionId, deadline) { function cleanupSessionJobs(cwd, sessionId, deadline) { if (!cwd || !sessionId) { - return { discoveryComplete: true, error: null }; + return { discoveryComplete: true, cwdBrokerTeardownSafe: true, error: null }; } let cleanupError = null; + let jobCleanupComplete = true; + let cwdBrokerTeardownSafe = true; let workspaceIndex = 0; const discovery = findSessionJobWorkspaces(cwd, sessionId, deadline); for (const workspaceRoot of discovery.workspaceRoots) { @@ -161,12 +173,27 @@ function cleanupSessionJobs(cwd, sessionId, deadline) { } workspaceIndex += 1; try { - cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline); + const cleanup = cleanupWorkspaceSessionJobs(workspaceRoot, sessionId, deadline); + cleanupError ??= cleanup.error; + if (!cleanup.brokerTeardownSafe) { + jobCleanupComplete = false; + if (workspaceIndex === 1) { + cwdBrokerTeardownSafe = false; + } + } } catch (error) { cleanupError ??= error; + jobCleanupComplete = false; + if (workspaceIndex === 1) { + cwdBrokerTeardownSafe = false; + } } } - return { discoveryComplete: discovery.complete, error: cleanupError }; + return { + discoveryComplete: discovery.complete && jobCleanupComplete, + cwdBrokerTeardownSafe, + error: cleanupError + }; } function handleSessionStart(input) { @@ -181,13 +208,16 @@ export async function handleSessionEnd(input) { const cleanupDeadline = Date.now() + SESSION_END_CLEANUP_BUDGET_MS; let cleanupError = null; let jobDiscoveryComplete = true; + let cwdBrokerTeardownSafe = true; try { const cleanup = cleanupSessionJobs(cwd, sessionId, cleanupDeadline); cleanupError = cleanup.error; jobDiscoveryComplete = cleanup.discoveryComplete; + cwdBrokerTeardownSafe = cleanup.cwdBrokerTeardownSafe; } catch (error) { cleanupError = error; jobDiscoveryComplete = false; + cwdBrokerTeardownSafe = false; } if (sessionId && jobDiscoveryComplete) { @@ -203,17 +233,19 @@ export async function handleSessionEnd(input) { } } - await teardownBrokerForCwd(cwd, sessionId, { - fallbackSession: process.env[BROKER_ENDPOINT_ENV] - ? { - endpoint: process.env[BROKER_ENDPOINT_ENV], - pidFile: process.env[PID_FILE_ENV] ?? null, - logFile: process.env[LOG_FILE_ENV] ?? null - } - : null, - killProcess: terminateProcessTree, - lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS - }); + if (cwdBrokerTeardownSafe) { + await teardownBrokerForCwd(cwd, sessionId, { + fallbackSession: process.env[BROKER_ENDPOINT_ENV] + ? { + endpoint: process.env[BROKER_ENDPOINT_ENV], + pidFile: process.env[PID_FILE_ENV] ?? null, + logFile: process.env[LOG_FILE_ENV] ?? null + } + : null, + killProcess: terminateProcessTree, + lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS + }); + } if (cleanupError) { throw cleanupError; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index dca66d9de..e1b3d4f3f 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -14,6 +14,7 @@ import { resolveJobFile, resolveJobLogFile, resolveStateDir, + resolveStateFile, resolveStateRoot, saveState, writeJobFile @@ -249,19 +250,6 @@ test("teardownBrokersForSession tears down a broker registered for a different c }); }); -test("teardownBrokersForSession leaves non-matching sessionId brokers intact", async () => { - await withPluginData(async () => { - const stateRoot = stateRootForTest(); - const brokerJson = writeBrokerJson(stateRoot, "other-1111111111111111", { - endpoint: "unix:/tmp/codex-test-nonexistent2.sock", - pidFile: null, logFile: null, sessionDir: null, pid: null, sessionId: "S" - }); - const count = await teardownBrokersForSession("OTHER", { killProcess: () => {} }); - assert.equal(count, 0); - assert.equal(fs.existsSync(brokerJson), true); - }); -}); - test("teardownBrokersForSession ignores broker.json without sessionId (legacy)", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); @@ -464,6 +452,78 @@ test("handleSessionEnd still tears down session brokers when job cleanup fails", }); }); +test("handleSessionEnd preserves session brokers when job state locking fails", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const otherWorkspace = makeTempDir(); + const stateFile = resolveStateFile(cwd); + const stateLock = `${stateFile}.lock`; + saveState(cwd, { + jobs: [{ + id: "locked-job", + status: "running", + sessionId: "S", + workspaceRoot: cwd, + pid: 999999999 + }] + }); + saveBrokerSession(cwd, { + endpoint: "unix:/tmp/codex-test-locked-job-cwd.sock", + sessionId: "S", + sessionIds: ["S"] + }); + saveBrokerSession(otherWorkspace, { + endpoint: "unix:/tmp/codex-test-locked-job-other.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + const originalRenameSync = fs.renameSync; + fs.renameSync = (source, destination) => { + if (destination === stateLock) { + throw Object.assign(new Error("state lock denied"), { code: "EACCES" }); + } + return originalRenameSync.call(fs, source, destination); + }; + + try { + await assert.rejects(() => handleSessionEnd({ cwd, session_id: "S" }), { code: "EACCES" }); + } finally { + fs.renameSync = originalRenameSync; + } + + assert.deepEqual(loadState(cwd).jobs.map((job) => job.id), ["locked-job"]); + assert.notEqual(loadBrokerSession(cwd), null); + assert.notEqual(loadBrokerSession(otherWorkspace), null); + }); +}); + +test("handleSessionEnd preserves session brokers when cwd job state is unreadable", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const otherWorkspace = makeTempDir(); + const stateFile = resolveStateFile(cwd); + fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + fs.writeFileSync(stateFile, "{not-json", "utf8"); + saveBrokerSession(cwd, { + endpoint: "unix:/tmp/codex-test-unreadable-job-cwd.sock", + sessionId: "S", + sessionIds: ["S"] + }); + saveBrokerSession(otherWorkspace, { + endpoint: "unix:/tmp/codex-test-unreadable-job-other.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + await handleSessionEnd({ cwd, session_id: "S" }); + + assert.equal(fs.readFileSync(stateFile, "utf8"), "{not-json"); + assert.notEqual(loadBrokerSession(cwd), null); + assert.notEqual(loadBrokerSession(otherWorkspace), null); + }); +}); + test("handleSessionEnd cleans session jobs from a different --cwd workspace before broker teardown", async () => { await withPluginData(async () => { const hookCwd = makeTempDir(); diff --git a/tests/state.test.mjs b/tests/state.test.mjs index 756e56ec9..45979e843 100644 --- a/tests/state.test.mjs +++ b/tests/state.test.mjs @@ -179,17 +179,17 @@ test("saveState does not release a state lock that has a replacement owner", () fs.rmSync(lockDir, { recursive: true, force: true }); }); -test("saveState reclaims an aged lock whose owner process exited", () => { +test("saveState immediately reclaims a fresh lock whose owner process exited", () => { const workspace = makeTempDir(); const stateFile = resolveStateFile(workspace); const lockDir = `${stateFile}.lock`; fs.mkdirSync(lockDir, { recursive: true }); fs.writeFileSync(path.join(lockDir, "owner"), "2147483647-crashed", "utf8"); - const old = new Date(Date.now() - 120000); - fs.utimesSync(lockDir, old, old); + const startedAt = Date.now(); saveState(workspace, { jobs: [] }); + assert.ok(Date.now() - startedAt < 1000); assert.equal(fs.existsSync(lockDir), false); assert.deepEqual(loadState(workspace).jobs, []); }); From 1eac442ac626a4e0e3890ba5472796ab7ebaff92 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 18:17:52 +0900 Subject: [PATCH 18/20] [COMMON] Close broker cleanup race windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 브로커 수명주기 정리 강화: - 종료된 세션의 spawn 및 재사용을 marker로 차단 - lock, scan, deadline 실패를 불완전 정리로 전파 - 세션 종료 동시성 회귀 테스트 확장 --- plugins/codex/scripts/lib/app-server.mjs | 20 +- .../codex/scripts/lib/broker-lifecycle.mjs | 427 +++++++++---- .../codex/scripts/session-lifecycle-hook.mjs | 45 +- tests/broker-lifecycle.test.mjs | 559 +++++++++++++++++- tests/runtime.test.mjs | 155 +++-- 5 files changed, 981 insertions(+), 225 deletions(-) diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 72b30a764..6c9e141ca 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -13,7 +13,7 @@ import process from "node:process"; import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; +import { ensureBrokerSession, resolveSessionId, reuseBrokerSession } from "./broker-lifecycle.mjs"; import { terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); @@ -337,11 +337,21 @@ export class CodexAppServerClient { let brokerEndpoint = null; if (!options.disableBroker) { brokerEndpoint = options.brokerEndpoint ?? options.env?.[BROKER_ENDPOINT_ENV] ?? process.env[BROKER_ENDPOINT_ENV] ?? null; - if (!brokerEndpoint && options.reuseExistingBroker) { - brokerEndpoint = loadBrokerSession(cwd)?.endpoint ?? null; + const sessionId = resolveSessionId({ env: options.env }); + if (!brokerEndpoint && sessionId && options.reuseExistingBroker) { + const brokerSession = await reuseBrokerSession(cwd, { + ...options.brokerOptions, + env: options.env, + killProcess: terminateProcessTree + }); + brokerEndpoint = brokerSession?.endpoint ?? null; } - if (!brokerEndpoint && !options.reuseExistingBroker) { - const brokerSession = await ensureBrokerSession(cwd, { env: options.env }); + if (!brokerEndpoint && sessionId && !options.reuseExistingBroker) { + const brokerSession = await ensureBrokerSession(cwd, { + ...options.brokerOptions, + env: options.env, + killProcess: terminateProcessTree + }); brokerEndpoint = brokerSession?.endpoint ?? null; } } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 8083162fc..ecab46367 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -13,12 +13,14 @@ export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; -const BROKER_STATE_LOCK_STALE_MS = 30000; const BROKER_STATE_LOCK_TIMEOUT_MS = 5000; const BROKER_SHUTDOWN_TIMEOUT_MS = 1000; const BROKER_LOCK_TIMEOUT_CODE = "EBROKERSTATELOCKTIMEOUT"; +export const BROKER_OWNER_ENDED_CODE = "EBROKEROWNERENDED"; +export const BROKER_CLEANUP_INCOMPLETE_CODE = "EBROKERCLEANUPINCOMPLETE"; const MAX_BROKER_STATE_BYTES = 64 * 1024; const ENDED_OWNER_MARKER_PREFIX = `${BROKER_STATE_FILE}.ended-`; +const BROKER_LOCK_SESSION_FILE = "session"; let brokerLockTokenSeq = 0; @@ -64,6 +66,22 @@ function isBrokerLockTimeout(error) { return error?.code === BROKER_LOCK_TIMEOUT_CODE; } +function brokerOwnerEndedError(sessionId) { + return Object.assign( + new Error(`Broker owner session ${sessionId ?? "unknown"} ended while the broker was starting.`), + { code: BROKER_OWNER_ENDED_CODE } + ); +} + +function brokerCleanupIncompleteError(reason, count = 0, cause = null) { + return Object.assign(new Error(`Broker cleanup stopped before completion: ${reason}.`), { + code: BROKER_CLEANUP_INCOMPLETE_CODE, + count, + reason, + ...(cause ? { cause } : {}) + }); +} + function isLockOwnerAlive(token) { const pid = Number.parseInt(token?.split("-", 1)[0] ?? "", 10); if (!Number.isSafeInteger(pid) || pid <= 0) { @@ -166,7 +184,6 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { const tokenFile = path.join(lockDir, "owner"); const token = `${process.pid}-${brokerLockTokenSeq += 1}`; const timeoutMs = options.timeoutMs ?? BROKER_STATE_LOCK_TIMEOUT_MS; - const staleMs = options.staleMs ?? BROKER_STATE_LOCK_STALE_MS; const deadline = Date.now() + timeoutMs; fs.mkdirSync(path.dirname(stateFile), { recursive: true }); @@ -187,6 +204,13 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { } candidate = fs.mkdtempSync(`${lockDir}.candidate-${process.pid}-`); fs.writeFileSync(path.join(candidate, "owner"), token, { encoding: "utf8", flag: "wx" }); + const ownerSessionId = options.ownerSessionId; + if (ownerSessionId && Buffer.byteLength(ownerSessionId, "utf8") <= 1024) { + fs.writeFileSync(path.join(candidate, BROKER_LOCK_SESSION_FILE), ownerSessionId, { + encoding: "utf8", + flag: "wx" + }); + } fs.renameSync(candidate, lockDir); candidate = null; if (recoverBrokerReclaimBarriers(lockDir)) { @@ -237,7 +261,7 @@ async function withBrokerStateFileLock(stateFile, fn, options = {}) { } stat = null; } - if (stat && Date.now() - stat.mtimeMs > staleMs) { + if (stat) { let ownerToken = null; try { ownerToken = fs.readFileSync(tokenFile, "utf8"); @@ -421,6 +445,36 @@ function readBrokerStateFile(stateFile) { } } +function readBrokerLockSession(lockDir) { + const sessionFile = path.join(lockDir, BROKER_LOCK_SESSION_FILE); + let descriptor = null; + try { + const before = fs.lstatSync(sessionFile); + if (!before.isFile() || before.size > 1024) { + return null; + } + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + descriptor = fs.openSync(sessionFile, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.size > 1024 || + opened.dev !== before.dev || opened.ino !== before.ino) { + return null; + } + return readBoundedUtf8(descriptor, 1024); + } catch { + return null; + } finally { + if (descriptor != null) { + try { + fs.closeSync(descriptor); + } catch { + // Ignore a descriptor already closed by a concurrent test shim. + } + } + } +} + export function loadBrokerSession(cwd) { return readBrokerStateFile(resolveBrokerStateFile(cwd)); } @@ -487,10 +541,10 @@ function applyEndedBrokerOwners(stateFile, session) { } } if (endedOwners.size === 0) { - return { session, markerFiles }; + return { session, markerFiles, endedOwners }; } if (!session) { - return { session, markerFiles }; + return { session, markerFiles, endedOwners }; } const owners = brokerSessionOwners(session).filter((owner) => !endedOwners.has(owner)); return { @@ -499,7 +553,8 @@ function applyEndedBrokerOwners(stateFile, session) { sessionId: owners[0] ?? null, sessionIds: owners }, - markerFiles + markerFiles, + endedOwners }; } @@ -608,12 +663,28 @@ function discardBrokerSession(cwd, session, options) { clearBrokerSession(cwd); } +function rejectEndedBrokerOwner(cwd, pending, sessionId, options) { + if (!sessionId || !pending.endedOwners.has(sessionId)) { + return; + } + const remainingOwners = brokerSessionOwners(pending.session); + if (pending.session && remainingOwners.length > 0) { + saveBrokerSession(cwd, pending.session); + } else { + discardBrokerSession(cwd, pending.session, options); + } + clearEndedOwnerMarkers(pending.markerFiles); + throw brokerOwnerEndedError(sessionId); +} + // Adopt cwd's recorded broker for this session if it is still live, else spawn // and persist a fresh one. Callers run this inside the state-file lock so two // racing sessions cannot each leave an orphaned broker with no broker.json. async function adoptOrSpawnBroker(cwd, options) { const stateFile = resolveBrokerStateFile(cwd); + const sessionId = resolveSessionId(options); const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + rejectEndedBrokerOwner(cwd, pending, sessionId, options); const current = pending.session; if (current && (await isBrokerEndpointReady(current.endpoint))) { const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); @@ -628,11 +699,31 @@ async function adoptOrSpawnBroker(cwd, options) { const session = await spawnReadyBroker(cwd, options); if (!session) { + const markerFile = sessionId ? endedOwnerMarkerFile(stateFile, sessionId) : null; + if (markerFile && fs.existsSync(markerFile)) { + clearEndedOwnerMarkers([markerFile]); + throw brokerOwnerEndedError(sessionId); + } return null; } - const withOwner = withBrokerSessionOwner(session, session.sessionId); - saveBrokerSession(cwd, withOwner); - return withOwner; + const spawnedWithOwner = withBrokerSessionOwner(session, session.sessionId); + const pendingAfterSpawn = applyEndedBrokerOwners(stateFile, spawnedWithOwner); + const activeOwners = brokerSessionOwners(pendingAfterSpawn.session); + if (activeOwners.length === 0) { + teardownBrokerSession({ + endpoint: session.endpoint, + pidFile: session.pidFile, + logFile: session.logFile, + sessionDir: session.sessionDir, + pid: session.pid, + killProcess: options.killProcess ?? null + }); + clearEndedOwnerMarkers(pendingAfterSpawn.markerFiles); + throw brokerOwnerEndedError(session.sessionId); + } + saveBrokerSession(cwd, pendingAfterSpawn.session); + clearEndedOwnerMarkers(pendingAfterSpawn.markerFiles); + return pendingAfterSpawn.session; } export async function ensureBrokerSession(cwd, options = {}) { @@ -649,6 +740,7 @@ export async function ensureBrokerSession(cwd, options = {}) { } const reused = await withBrokerStateFileLock(stateFile, () => { const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + rejectEndedBrokerOwner(cwd, pending, resolveSessionId(options), options); const current = pending.session; if (!current || current.endpoint !== existing.endpoint) { return null; @@ -670,7 +762,45 @@ export async function ensureBrokerSession(cwd, options = {}) { // Slow path: adopt-or-spawn under the lock so concurrent spawns don't orphan // brokers. resolveStateDir must exist before we can create the lock dir. fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); - return withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options), lockOptions); + return withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options), { + ...lockOptions, + ownerSessionId: resolveSessionId(options) + }); +} + +// Reuse cwd's recorded broker without claiming ownership or spawning a new +// broker. Probe-only callers still have to honor ended-owner markers; reading +// broker.json directly can otherwise let an ended session reconnect after its +// teardown was deferred by lock contention. +export async function reuseBrokerSession(cwd, options = {}) { + const stateFile = resolveBrokerStateFile(cwd); + // Always take the state lock, even when broker.json has not been published. + // SessionEnd can intentionally leave only an ended-owner marker when it + // races before the first broker lock; probe-only reuse must reconcile that + // marker before it is allowed to fall back to a direct app-server. + fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); + const lockOptions = options.lockTimeoutMs == null ? {} : { timeoutMs: options.lockTimeoutMs }; + return withBrokerStateFileLock(stateFile, async () => { + const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + rejectEndedBrokerOwner(cwd, pending, resolveSessionId(options), options); + + const current = pending.session; + if (!current || brokerSessionOwners(current).length === 0) { + discardBrokerSession(cwd, current, options); + clearEndedOwnerMarkers(pending.markerFiles); + return null; + } + if (!(await isBrokerEndpointReady(current.endpoint))) { + discardBrokerSession(cwd, current, options); + clearEndedOwnerMarkers(pending.markerFiles); + return null; + } + if (pending.markerFiles.length > 0) { + saveBrokerSession(cwd, current); + } + clearEndedOwnerMarkers(pending.markerFiles); + return current; + }, lockOptions); } export async function teardownBrokerForCwd( @@ -684,7 +814,14 @@ export async function teardownBrokerForCwd( } = {} ) { const stateFile = resolveBrokerStateFile(cwd); - if (!fs.existsSync(stateFile) && !fallbackSession) { + const lockDir = `${stateFile}.lock`; + if (!fs.existsSync(stateFile) && !fs.existsSync(lockDir) && !fallbackSession) { + if (sessionId) { + // Persist the end intent even before a concurrent first broker start has + // published its lock. The starter will reconcile this marker before it + // can spawn or persist ownership for the ended session. + recordEndedBrokerOwner(stateFile, sessionId); + } return false; } fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); @@ -742,7 +879,7 @@ export async function teardownBrokerForCwd( ); } catch (error) { if (isBrokerLockTimeout(error)) { - return false; + throw brokerCleanupIncompleteError("lock-timeout"); } throw error; } @@ -767,140 +904,168 @@ export async function teardownBrokersForSession( } const deadline = budgetMs != null ? Date.now() + budgetMs : null; + if (deadline != null && Date.now() >= deadline) { + throw brokerCleanupIncompleteError("deadline"); + } let count = 0; let teardownError = null; + let incompleteReason = null; const excludedStateFile = excludeCwd ? resolveBrokerStateFile(excludeCwd) : null; - for (const entry of fs.readdirSync(stateRoot, { withFileTypes: true })) { - if (deadline != null && Date.now() >= deadline) { - break; - } - if (!entry.isDirectory() || entry.isSymbolicLink()) { - continue; - } - const stateFile = path.join(stateRoot, entry.name, BROKER_STATE_FILE); - if (stateFile === excludedStateFile) { - continue; - } - if (!fs.existsSync(stateFile)) { - continue; - } - - // Ownership pre-check without the lock: never block on a lock held for a - // workspace this session does not own. The owner set only ever grows to - // include our sessionId (reuse) or shrinks when we ourselves remove it, so - // an unlocked read cannot falsely exclude a broker we own. A parse failure - // (e.g. a genuinely corrupt file) is NOT treated as "not ours" — fall - // through to the locked re-read, which is authoritative, rather than - // skipping a broker that might belong to this session. - let preview = null; - try { - preview = readBrokerStateFile(stateFile); - } catch { - preview = null; - } - if (preview && !brokerSessionOwners(preview).includes(sessionId)) { - continue; - } - recordEndedBrokerOwner(stateFile, sessionId); + const stateDirectory = fs.opendirSync(stateRoot); + try { + while (true) { + if (deadline != null && Date.now() >= deadline) { + incompleteReason = "deadline"; + break; + } + const entry = stateDirectory.readSync(); + if (!entry) { + break; + } + if (deadline != null && Date.now() >= deadline) { + incompleteReason = "deadline"; + break; + } + if (!entry.isDirectory() || entry.isSymbolicLink()) { + continue; + } + const stateFile = path.join(stateRoot, entry.name, BROKER_STATE_FILE); + if (stateFile === excludedStateFile) { + continue; + } + const stateExists = fs.existsSync(stateFile); + const lockDir = `${stateFile}.lock`; + if (!stateExists && readBrokerLockSession(lockDir) !== sessionId) { + continue; + } - const remaining = deadline != null ? deadline - Date.now() : lockTimeoutMs; - if (remaining <= 0) { - break; - } - const entryLockTimeout = Math.min(lockTimeoutMs, remaining); + // Ownership pre-check without the lock: never block on a lock held for a + // workspace this session does not own. The owner set only ever grows to + // include our sessionId (reuse) or shrinks when we ourselves remove it, so + // an unlocked read cannot falsely exclude a broker we own. A parse failure + // (e.g. a genuinely corrupt file) is NOT treated as "not ours" — fall + // through to the locked re-read, which is authoritative, rather than + // skipping a broker that might belong to this session. + let preview = null; + if (stateExists) { + try { + preview = readBrokerStateFile(stateFile); + } catch { + preview = null; + } + } + if (preview && !brokerSessionOwners(preview).includes(sessionId)) { + continue; + } + try { + recordEndedBrokerOwner(stateFile, sessionId); + } catch (error) { + teardownError ??= error; + continue; + } - try { - await withBrokerStateFileLock( - stateFile, - async () => { - if (!fs.existsSync(stateFile)) { - return; - } + const remaining = deadline != null ? deadline - Date.now() : lockTimeoutMs; + if (remaining <= 0) { + incompleteReason = "deadline"; + break; + } + const entryLockTimeout = Math.min(lockTimeoutMs, remaining); - const pending = applyEndedBrokerOwners(stateFile, readBrokerStateFile(stateFile)); - const session = pending.session; - if (!session) { - clearEndedOwnerMarkers(pending.markerFiles); - return; - } - const owners = brokerSessionOwners(session); - if (!owners.includes(sessionId)) { - if (pending.markerFiles.length > 0) { - if (owners.length === 0) { - if (session.endpoint) { - const shutdownWait = - deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; - if (shutdownWait > 0) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + try { + await withBrokerStateFileLock( + stateFile, + async () => { + const pending = applyEndedBrokerOwners(stateFile, readBrokerStateFile(stateFile)); + const session = pending.session; + if (!session) { + clearEndedOwnerMarkers(pending.markerFiles); + return; + } + const owners = brokerSessionOwners(session); + if (!owners.includes(sessionId)) { + if (pending.markerFiles.length > 0) { + if (owners.length === 0) { + if (session.endpoint) { + const shutdownWait = + deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; + if (shutdownWait > 0) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + } } + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + count += 1; + } else { + writeBrokerStateFile(stateFile, session); } - teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, - killProcess - }); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); - } - count += 1; - } else { - writeBrokerStateFile(stateFile, session); + clearEndedOwnerMarkers(pending.markerFiles); } + return; + } + const remainingOwners = owners.filter((owner) => owner !== sessionId); + if (remainingOwners.length > 0) { + writeBrokerStateFile(stateFile, { + ...session, + sessionId: remainingOwners[0], + sessionIds: remainingOwners + }); clearEndedOwnerMarkers(pending.markerFiles); + return; } - return; - } - const remainingOwners = owners.filter((owner) => owner !== sessionId); - if (remainingOwners.length > 0) { - writeBrokerStateFile(stateFile, { - ...session, - sessionId: remainingOwners[0], - sessionIds: remainingOwners - }); - clearEndedOwnerMarkers(pending.markerFiles); - return; - } - // Bound the graceful-shutdown wait by the remaining scan budget, not - // just the per-RPC default: several unresponsive endpoints could - // otherwise each burn shutdownTimeoutMs and push the whole scan past - // the SessionEnd hook's budget. Out of budget → skip the RPC and let - // teardownBrokerSession terminate the process directly. - if (session.endpoint) { - const shutdownWait = - deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; - if (shutdownWait > 0) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + // Bound the graceful-shutdown wait by the remaining scan budget, not + // just the per-RPC default: several unresponsive endpoints could + // otherwise each burn shutdownTimeoutMs and push the whole scan past + // the SessionEnd hook's budget. Out of budget → skip the RPC and let + // teardownBrokerSession terminate the process directly. + if (session.endpoint) { + const shutdownWait = + deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; + if (shutdownWait > 0) { + await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + } } - } - teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, - killProcess - }); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); - } - clearEndedOwnerMarkers(pending.markerFiles); - count += 1; - }, - { timeoutMs: entryLockTimeout } - ); - } catch (error) { - if (!isBrokerLockTimeout(error)) { - teardownError ??= error; - continue; + teardownBrokerSession({ + endpoint: session.endpoint ?? null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + pid: session.pid ?? null, + killProcess + }); + if (fs.existsSync(stateFile)) { + fs.unlinkSync(stateFile); + } + clearEndedOwnerMarkers(pending.markerFiles); + count += 1; + }, + { timeoutMs: entryLockTimeout } + ); + } catch (error) { + if (!isBrokerLockTimeout(error)) { + teardownError ??= error; + continue; + } + // This entry's lock remained unavailable within the timeout. Skip it so + // the rest of this session's brokers still get torn down instead of + // aborting the whole scan, but report that cleanup was incomplete. + incompleteReason ??= "lock-timeout"; } - // This entry's lock remained unavailable within the timeout. Skip it so - // the rest of this session's brokers still get torn down instead of - // aborting the whole scan. } + } finally { + stateDirectory.closeSync(); + } + if (incompleteReason) { + throw brokerCleanupIncompleteError(incompleteReason, count, teardownError); } if (teardownError) { throw teardownError; diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 5cbec7a3c..cf0dc9156 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url"; import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; import { + BROKER_CLEANUP_INCOMPLETE_CODE, LOG_FILE_ENV, PID_FILE_ENV, teardownBrokerForCwd, @@ -24,9 +25,15 @@ const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; // teardown so the cwd fallback retains time before the hook is killed. const SESSION_END_CLEANUP_BUDGET_MS = 3000; const SESSION_END_LOCK_TIMEOUT_MS = 750; -const MAX_SESSION_JOB_STATE_FILES = 1000; const MAX_SESSION_JOB_STATE_BYTES = 1024 * 1024; +function cleanupIncompleteError(reason) { + return Object.assign(new Error(`Session cleanup stopped before scanning all job state: ${reason}.`), { + code: BROKER_CLEANUP_INCOMPLETE_CODE, + reason + }); +} + function readHookInput() { const raw = fs.readFileSync(0, "utf8").trim(); if (!raw) { @@ -121,14 +128,12 @@ function findSessionJobWorkspaces(cwd, sessionId, deadline) { let complete = true; let exhausted = false; try { - let scanned = 0; - while (scanned < MAX_SESSION_JOB_STATE_FILES && Date.now() < deadline) { + while (Date.now() < deadline) { const entry = stateDirectory.readSync(); if (!entry) { exhausted = true; break; } - scanned += 1; if (entry.isSymbolicLink()) { complete = false; continue; @@ -164,11 +169,16 @@ function cleanupSessionJobs(cwd, sessionId, deadline) { let cwdBrokerTeardownSafe = true; let workspaceIndex = 0; const discovery = findSessionJobWorkspaces(cwd, sessionId, deadline); + if (!discovery.complete) { + cleanupError = cleanupIncompleteError("job-discovery-incomplete"); + } for (const workspaceRoot of discovery.workspaceRoots) { // Always clean the hook cwd first. Bound additional workspace cleanup by // the shared SessionEnd deadline so broker cleanup and the cwd fallback // retain time inside the hook's 5-second limit. if (workspaceIndex > 0 && Date.now() >= deadline) { + jobCleanupComplete = false; + cleanupError ??= cleanupIncompleteError("job-cleanup-deadline"); break; } workspaceIndex += 1; @@ -177,6 +187,7 @@ function cleanupSessionJobs(cwd, sessionId, deadline) { cleanupError ??= cleanup.error; if (!cleanup.brokerTeardownSafe) { jobCleanupComplete = false; + cleanupError ??= cleanupIncompleteError("job-state-incomplete"); if (workspaceIndex === 1) { cwdBrokerTeardownSafe = false; } @@ -234,17 +245,21 @@ export async function handleSessionEnd(input) { } if (cwdBrokerTeardownSafe) { - await teardownBrokerForCwd(cwd, sessionId, { - fallbackSession: process.env[BROKER_ENDPOINT_ENV] - ? { - endpoint: process.env[BROKER_ENDPOINT_ENV], - pidFile: process.env[PID_FILE_ENV] ?? null, - logFile: process.env[LOG_FILE_ENV] ?? null - } - : null, - killProcess: terminateProcessTree, - lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS - }); + try { + await teardownBrokerForCwd(cwd, sessionId, { + fallbackSession: process.env[BROKER_ENDPOINT_ENV] + ? { + endpoint: process.env[BROKER_ENDPOINT_ENV], + pidFile: process.env[PID_FILE_ENV] ?? null, + logFile: process.env[LOG_FILE_ENV] ?? null + } + : null, + killProcess: terminateProcessTree, + lockTimeoutMs: SESSION_END_LOCK_TIMEOUT_MS + }); + } catch (error) { + cleanupError ??= error; + } } if (cleanupError) { throw cleanupError; diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index e1b3d4f3f..e4b47bb9c 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -20,6 +20,8 @@ import { writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; import { + BROKER_CLEANUP_INCOMPLETE_CODE, + BROKER_OWNER_ENDED_CODE, ensureBrokerSession, loadBrokerSession, resolveSessionId, @@ -28,6 +30,7 @@ import { teardownBrokerForCwd, teardownBrokersForSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { handleSessionEnd } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; function sleep(ms) { @@ -75,9 +78,11 @@ const args = process.argv.slice(2); const get = (name) => args[args.indexOf(name) + 1]; const sockPath = get("--endpoint").replace(/^(?:unix|pipe):/, ""); const server = net.createServer((socket) => socket.end()); -server.listen(sockPath, () => { - fs.writeFileSync(get("--pid-file"), String(process.pid), "utf8"); -}); +setTimeout(() => { + server.listen(sockPath, () => { + fs.writeFileSync(get("--pid-file"), String(process.pid), "utf8"); + }); +}, Number(process.env.TEST_BROKER_LISTEN_DELAY_MS || 0)); `; function writeFakeBrokerScript() { @@ -498,7 +503,7 @@ test("handleSessionEnd preserves session brokers when job state locking fails", }); }); -test("handleSessionEnd preserves session brokers when cwd job state is unreadable", async () => { +test("handleSessionEnd reports incomplete cleanup when cwd job state is unreadable", async () => { await withPluginData(async () => { const cwd = makeTempDir(); const otherWorkspace = makeTempDir(); @@ -516,7 +521,10 @@ test("handleSessionEnd preserves session brokers when cwd job state is unreadabl sessionIds: ["S"] }); - await handleSessionEnd({ cwd, session_id: "S" }); + await assert.rejects( + () => handleSessionEnd({ cwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "job-discovery-incomplete" } + ); assert.equal(fs.readFileSync(stateFile, "utf8"), "{not-json"); assert.notEqual(loadBrokerSession(cwd), null); @@ -604,7 +612,61 @@ test("handleSessionEnd continues cross-cwd cleanup after one workspace cleanup f }); }); -test("handleSessionEnd preserves cross-cwd brokers when job-state discovery is incomplete", async () => { +test("handleSessionEnd reports incomplete cleanup when the deadline skips a discovered workspace", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const otherWorkspace = makeTempDir(); + saveState(hookCwd, { + jobs: [{ + id: "hook-job", + status: "completed", + sessionId: "S", + workspaceRoot: hookCwd + }] + }); + saveState(otherWorkspace, { + jobs: [{ + id: "other-job", + status: "completed", + sessionId: "S", + workspaceRoot: otherWorkspace + }] + }); + saveBrokerSession(otherWorkspace, { + endpoint: "unix:/tmp/codex-test-job-cleanup-deadline.sock", + sessionId: "S", + sessionIds: ["S"] + }); + + const hookStateFile = resolveStateFile(hookCwd); + const originalRenameSync = fs.renameSync; + const originalNow = Date.now; + let now = 0; + Date.now = () => now; + fs.renameSync = (source, destination) => { + const result = originalRenameSync.call(fs, source, destination); + if (destination === hookStateFile) { + now = 3000; + } + return result; + }; + + try { + await assert.rejects( + () => handleSessionEnd({ cwd: hookCwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "job-cleanup-deadline" } + ); + } finally { + Date.now = originalNow; + fs.renameSync = originalRenameSync; + } + + assert.deepEqual(loadState(otherWorkspace).jobs.map((job) => job.id), ["other-job"]); + assert.notEqual(loadBrokerSession(otherWorkspace), null); + }); +}); + +test("handleSessionEnd reports incomplete cleanup when job-state discovery is incomplete", async () => { await withPluginData(async () => { const hookCwd = makeTempDir(); const jobWorkspace = makeTempDir(); @@ -629,13 +691,28 @@ test("handleSessionEnd preserves cross-cwd brokers when job-state discovery is i sessionIds: ["S"] }); - await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + await assert.rejects( + () => handleSessionEnd({ cwd: hookCwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "job-discovery-incomplete" } + ); assert.equal(loadState(jobWorkspace).jobs.length, 1); assert.notEqual(loadBrokerSession(jobWorkspace), null); }); }); +test("handleSessionEnd does not cap a complete job-state discovery scan", async () => { + await withPluginData(async () => { + const hookCwd = makeTempDir(); + const stateRoot = stateRootForTest(); + for (let index = 0; index < 1001; index += 1) { + fs.mkdirSync(path.join(stateRoot, `empty-${String(index).padStart(4, "0")}`), { recursive: true }); + } + + await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + }); +}); + test("handleSessionEnd gives broker teardown only the remaining shared cleanup budget", async () => { await withPluginData(async () => { const hookCwd = makeTempDir(); @@ -662,7 +739,10 @@ test("handleSessionEnd gives broker teardown only the remaining shared cleanup b }; try { - await handleSessionEnd({ cwd: hookCwd, session_id: "S" }); + await assert.rejects( + () => handleSessionEnd({ cwd: hookCwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE } + ); } finally { Date.now = originalNow; dirPrototype.readSync = originalReadSync; @@ -704,7 +784,7 @@ test("teardownBrokersForSession times out unresponsive broker shutdown requests" }); }); -test("teardownBrokersForSession skips a same-session locked entry but tears down the rest", async () => { +test("teardownBrokersForSession reports a same-session locked entry after tearing down the rest", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { const stateRoot = stateRootForTest(); @@ -726,9 +806,16 @@ test("teardownBrokersForSession skips a same-session locked entry but tears down }); try { - const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }); - // The locked entry is skipped (not aborted), the reachable one is cleaned. - assert.equal(count, 1); + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }), + (error) => { + assert.equal(error.code, BROKER_CLEANUP_INCOMPLETE_CODE); + assert.equal(error.reason, "lock-timeout"); + assert.equal(error.count, 1); + return true; + } + ); + // The locked entry is skipped, the reachable one is still cleaned. assert.equal(fs.existsSync(cleanableJson), false); assert.equal(fs.existsSync(lockedJson), true); assert.equal(requests.length, 0); @@ -749,6 +836,92 @@ test("teardownBrokersForSession skips a same-session locked entry but tears down }); }); +test("ensureBrokerSession cannot re-own a broker after its session end marker is recorded", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "S", sessionIds: ["S"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + + try { + await assert.rejects( + () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + + await assert.rejects( + () => ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "S" } }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(requests.length, 0); + }); + }); +}); + +test("reuseExistingBroker cannot reconnect after its session end marker is recorded", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "S", sessionIds: ["S"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + + try { + await assert.rejects( + () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + + await assert.rejects( + () => CodexAppServerClient.connect(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "S" }, + reuseExistingBroker: true + }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(requests.length, 0); + }); + }); +}); + +test("reuseExistingBroker rejects a marker-only ended session before direct fallback", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + + assert.equal( + await teardownBrokerForCwd(cwd, "S", { killProcess: () => {} }), + false + ); + assert.equal(loadBrokerSession(cwd), null); + + await assert.rejects( + () => CodexAppServerClient.connect(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "S" }, + reuseExistingBroker: true + }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.equal(loadBrokerSession(cwd), null); + }); +}); + test("teardownBrokersForSession replaces an invalid non-directory lock path", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); @@ -791,8 +964,10 @@ test("teardownBrokersForSession never deletes a valid lock that replaces an inva }; try { - const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }); - assert.equal(count, 0); + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 60 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); } finally { fs.lstatSync = originalLstatSync; fs.rmSync(lockPath, { recursive: true, force: true }); @@ -865,7 +1040,32 @@ test("teardownBrokersForSession reclaims a stale lock and still tears the broker }); }); -test("teardownBrokersForSession stops scanning once its time budget is exhausted", async () => { +test("teardownBrokersForSession immediately reclaims a fresh dead-owner lock", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-freshdeadlock", { + endpoint: "unix:/tmp/codex-test-fresh-dead-lock.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "S" + }); + const lockDir = `${brokerJson}.lock`; + fs.mkdirSync(lockDir); + fs.writeFileSync(path.join(lockDir, "owner"), "2147483647-crashed", "utf8"); + + const startedAt = Date.now(); + const count = await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 200 }); + + assert.equal(count, 1); + assert.equal(fs.existsSync(brokerJson), false); + assert.equal(fs.existsSync(lockDir), false); + assert.ok(Date.now() - startedAt < 1000); + }); +}); + +test("teardownBrokersForSession reports an incomplete scan once its time budget is exhausted", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); // Two same-session entries, both with held locks. With a tiny budget the @@ -883,7 +1083,14 @@ test("teardownBrokersForSession stops scanning once its time budget is exhausted try { const startedAt = Date.now(); - await teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 5000, budgetMs: 200 }); + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 5000, budgetMs: 200 }), + (error) => { + assert.equal(error.code, BROKER_CLEANUP_INCOMPLETE_CODE); + assert.equal(error.reason, "deadline"); + return true; + } + ); const elapsed = Date.now() - startedAt; assert.ok(elapsed < 1500, `expected budget-bounded scan, took ${elapsed}ms`); } finally { @@ -893,6 +1100,62 @@ test("teardownBrokersForSession stops scanning once its time budget is exhausted }); }); +test("teardownBrokersForSession streams the state root instead of reading it all at once", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const brokerJson = writeBrokerJson(stateRoot, "worktree-streamedscan", { + endpoint: "unix:/tmp/codex-test-streamed-scan.sock", + pidFile: null, + logFile: null, + sessionDir: null, + pid: null, + sessionId: "S" + }); + const originalReaddirSync = fs.readdirSync; + fs.readdirSync = (target, ...args) => { + if (target === stateRoot) { + throw new Error("state root must be streamed"); + } + return originalReaddirSync.call(fs, target, ...args); + }; + + try { + assert.equal(await teardownBrokersForSession("S", { killProcess: () => {} }), 1); + } finally { + fs.readdirSync = originalReaddirSync; + } + + assert.equal(fs.existsSync(brokerJson), false); + }); +}); + +test("teardownBrokersForSession does not silently cap a complete state-root scan", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + fs.mkdirSync(stateRoot, { recursive: true }); + for (let index = 0; index < 1001; index += 1) { + fs.mkdirSync(path.join(stateRoot, `empty-${String(index).padStart(4, "0")}`)); + } + + const probe = fs.opendirSync(stateRoot); + const dirPrototype = Object.getPrototypeOf(probe); + probe.closeSync(); + const originalReadSync = dirPrototype.readSync; + let reads = 0; + dirPrototype.readSync = function (...args) { + reads += 1; + return originalReadSync.call(this, ...args); + }; + + try { + assert.equal(await teardownBrokersForSession("S", { killProcess: () => {} }), 0); + } finally { + dirPrototype.readSync = originalReadSync; + } + assert.ok(reads > 1000, `expected an uncapped scan, observed ${reads} reads`); + }); +}); + test("ensureBrokerSession spawns and persists a fresh broker when the recorded one is dead", async () => { await withPluginData(async () => { const cwd = makeTempDir(); @@ -958,6 +1221,167 @@ test("ensureBrokerSession never spawns without the state lock after acquisition }); }); +test("ensureBrokerSession does not persist a broker whose owner ended during spawn", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const spawnMarker = path.join(makeTempDir(), "spawned"); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + const killed = []; + const pending = ensureBrokerSession(cwd, { + scriptPath: writeFakeBrokerScript(), + env: { + ...process.env, + CODEX_COMPANION_SESSION_ID: "S", + TEST_BROKER_SPAWN_MARKER: spawnMarker, + TEST_BROKER_LISTEN_DELAY_MS: "200" + }, + killProcess(pid) { + killed.push(pid); + process.kill(pid); + } + }); + + const deadline = Date.now() + 2000; + while (!fs.existsSync(spawnMarker) && Date.now() < deadline) { + await sleep(10); + } + assert.equal(fs.existsSync(spawnMarker), true); + assert.equal(fs.existsSync(stateFile), false); + assert.equal(fs.existsSync(`${stateFile}.lock`), true); + + await assert.rejects( + () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + await assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); + assert.equal(loadBrokerSession(cwd), null); + assert.equal(fs.existsSync(`${stateFile}.lock`), false); + assert.equal(killed.length, 1); + }); +}); + +test("mismatched-cwd SessionEnd marks an in-flight broker spawn before broker.json exists", async () => { + await withPluginData(async () => { + const brokerCwd = makeTempDir(); + const hookCwd = makeTempDir(); + const spawnMarker = path.join(makeTempDir(), "spawned"); + const stateFile = path.join(resolveStateDir(brokerCwd), "broker.json"); + const killed = []; + const pending = ensureBrokerSession(brokerCwd, { + scriptPath: writeFakeBrokerScript(), + env: { + ...process.env, + CODEX_COMPANION_SESSION_ID: "S", + TEST_BROKER_SPAWN_MARKER: spawnMarker, + TEST_BROKER_LISTEN_DELAY_MS: "1000" + }, + killProcess(pid) { + killed.push(pid); + process.kill(pid); + } + }); + + const deadline = Date.now() + 2000; + while (!fs.existsSync(spawnMarker) && Date.now() < deadline) { + await sleep(10); + } + assert.equal(fs.existsSync(spawnMarker), true); + assert.equal(fs.existsSync(stateFile), false); + assert.equal(fs.existsSync(`${stateFile}.lock`), true); + + await assert.rejects( + () => handleSessionEnd({ cwd: hookCwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + await assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); + assert.equal(loadBrokerSession(brokerCwd), null); + assert.equal(fs.existsSync(`${stateFile}.lock`), false); + assert.equal(killed.length, 1); + }); +}); + +test("CodexAppServerClient does not fall back to a direct app-server after its broker owner ends", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const spawnMarker = path.join(makeTempDir(), "broker-spawned"); + const directMarker = path.join(makeTempDir(), "direct-spawned"); + const fakeBin = makeTempDir(); + const fakeCodex = path.join(fakeBin, "codex"); + fs.writeFileSync( + fakeCodex, + `#!/bin/sh\nprintf spawned > "${directMarker}"\nexit 1\n`, + { encoding: "utf8", mode: 0o755 } + ); + + const env = { + ...process.env, + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + CODEX_COMPANION_SESSION_ID: "S", + TEST_BROKER_SPAWN_MARKER: spawnMarker, + TEST_BROKER_LISTEN_DELAY_MS: "200" + }; + const pending = CodexAppServerClient.connect(cwd, { + env, + brokerOptions: { scriptPath: writeFakeBrokerScript() } + }); + + const deadline = Date.now() + 2000; + while (!fs.existsSync(spawnMarker) && Date.now() < deadline) { + await sleep(10); + } + assert.equal(fs.existsSync(spawnMarker), true); + await assert.rejects( + () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + + await assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); + assert.equal(fs.existsSync(directMarker), false); + assert.equal(loadBrokerSession(cwd), null); + }); +}); + +test("CodexAppServerClient does not fall back after an ended owner's broker spawn times out", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const spawnMarker = path.join(makeTempDir(), "broker-spawned"); + const directMarker = path.join(makeTempDir(), "direct-spawned"); + const fakeBin = makeTempDir(); + const fakeCodex = path.join(fakeBin, "codex"); + fs.writeFileSync( + fakeCodex, + `#!/bin/sh\nprintf spawned > "${directMarker}"\nexit 1\n`, + { encoding: "utf8", mode: 0o755 } + ); + + const pending = CodexAppServerClient.connect(cwd, { + env: { + ...process.env, + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + CODEX_COMPANION_SESSION_ID: "S", + TEST_BROKER_SPAWN_MARKER: spawnMarker, + TEST_BROKER_LISTEN_DELAY_MS: "1000" + }, + brokerOptions: { scriptPath: writeFakeBrokerScript(), timeoutMs: 100 } + }); + const rejected = assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); + + const deadline = Date.now() + 2000; + while (!fs.existsSync(spawnMarker) && Date.now() < deadline) { + await sleep(10); + } + assert.equal(fs.existsSync(spawnMarker), true); + await assert.rejects( + () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + + await rejected; + assert.equal(fs.existsSync(directMarker), false); + assert.equal(loadBrokerSession(cwd), null); + }); +}); + test("ensureBrokerSession does not resurrect a broker torn down while waiting for the state lock", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, sessionDir }) => { @@ -1109,7 +1533,7 @@ test("sendBrokerShutdown returns immediately for a non-positive timeout", async }); }); -test("teardownBrokerForCwd does not create state for an unused workspace", async () => { +test("teardownBrokerForCwd records an ended owner before the first broker lock exists", async () => { await withPluginData(async () => { const cwd = makeTempDir(); const stateDir = resolveStateDir(cwd); @@ -1117,7 +1541,12 @@ test("teardownBrokerForCwd does not create state for an unused workspace", async const tornDown = await teardownBrokerForCwd(cwd, "S"); assert.equal(tornDown, false); - assert.equal(fs.existsSync(stateDir), false); + assert.equal(fs.existsSync(stateDir), true); + await assert.rejects( + () => ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "S" } }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.equal(loadBrokerSession(cwd), null); }); }); @@ -1221,6 +1650,82 @@ test("teardownBrokersForSession continues after one broker cleanup fails", async }); }); +test("teardownBrokersForSession continues after one ended-owner marker write fails", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const firstJson = writeBrokerJson(stateRoot, "worktree-0000markerfail", { + endpoint: "unix:/tmp/codex-test-marker-failure.sock", + sessionId: "S" + }); + const secondJson = writeBrokerJson(stateRoot, "worktree-9999markersuccess", { + endpoint: "unix:/tmp/codex-test-marker-success.sock", + sessionId: "S" + }); + const firstDir = path.dirname(firstJson); + const originalWriteFileSync = fs.writeFileSync.bind(fs); + fs.writeFileSync = (file, ...args) => { + if (path.dirname(String(file)) === firstDir && path.basename(String(file)).startsWith("broker.json.ended-")) { + throw Object.assign(new Error("simulated marker write failure"), { code: "EACCES" }); + } + return originalWriteFileSync(file, ...args); + }; + + try { + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {} }), + { code: "EACCES" } + ); + } finally { + fs.writeFileSync = originalWriteFileSync; + } + + assert.equal(fs.existsSync(firstJson), true); + assert.equal(fs.existsSync(secondJson), false); + }); +}); + +test("teardownBrokersForSession preserves incomplete status when later cleanup also fails", async () => { + await withPluginData(async () => { + const stateRoot = stateRootForTest(); + const lockedJson = writeBrokerJson(stateRoot, "worktree-0000locked", { + endpoint: "unix:/tmp/codex-test-locked-first.sock", + sessionId: "S" + }); + const failedJson = writeBrokerJson(stateRoot, "worktree-9999markerfail", { + endpoint: "unix:/tmp/codex-test-marker-failure-later.sock", + sessionId: "S" + }); + const failedDir = path.dirname(failedJson); + const lockDir = `${lockedJson}.lock`; + fs.mkdirSync(lockDir); + const originalWriteFileSync = fs.writeFileSync.bind(fs); + fs.writeFileSync = (file, ...args) => { + if (path.dirname(String(file)) === failedDir && path.basename(String(file)).startsWith("broker.json.ended-")) { + throw Object.assign(new Error("simulated later marker failure"), { code: "EACCES" }); + } + return originalWriteFileSync(file, ...args); + }; + + try { + await assert.rejects( + () => teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 50 }), + (error) => { + assert.equal(error.code, BROKER_CLEANUP_INCOMPLETE_CODE); + assert.equal(error.reason, "lock-timeout"); + assert.equal(error.cause?.code, "EACCES"); + return true; + } + ); + } finally { + fs.writeFileSync = originalWriteFileSync; + fs.rmSync(lockDir, { recursive: true, force: true }); + } + + assert.equal(fs.existsSync(lockedJson), true); + assert.equal(fs.existsSync(failedJson), true); + }); +}); + test("handleSessionEnd skips the cwd fallback while broker state remains locked", async () => { await withPluginData(async () => { await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { @@ -1235,7 +1740,10 @@ test("handleSessionEnd skips the cwd fallback while broker state remains locked" fs.mkdirSync(lockDir); try { - await handleSessionEnd({ cwd, session_id: "S" }); + await assert.rejects( + () => handleSessionEnd({ cwd, session_id: "S" }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); assert.notEqual(loadBrokerSession(cwd), null); assert.equal(requests.length, 0); } finally { @@ -1304,11 +1812,14 @@ test("teardownBrokersForSession caps shutdown waits to the remaining budget", as const startedAt = Date.now(); // Endpoints accept but never reply; a per-RPC 1s wait on each would blow // the budget. The scan must honor budgetMs across shutdown waits. - await teardownBrokersForSession("S", { - killProcess: () => {}, - shutdownTimeoutMs: 1000, - budgetMs: 300 - }); + await assert.rejects( + () => teardownBrokersForSession("S", { + killProcess: () => {}, + shutdownTimeoutMs: 1000, + budgetMs: 300 + }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE } + ); const elapsed = Date.now() - startedAt; assert.ok(elapsed < 900, `expected budget-capped shutdown, took ${elapsed}ms`); assert.equal(Number(fs.existsSync(first)) + Number(fs.existsSync(second)), 1); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index af99274a0..b3c4f6be4 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -7,7 +7,13 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir, run } from "./helpers.mjs"; -import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { + loadBrokerSession, + saveBrokerSession, + teardownBrokerForCwd, + teardownBrokersForSession +} from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -15,6 +21,37 @@ const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); const SCRIPT = path.join(PLUGIN_ROOT, "scripts", "codex-companion.mjs"); const STOP_HOOK = path.join(PLUGIN_ROOT, "scripts", "stop-review-gate-hook.mjs"); const SESSION_HOOK = path.join(PLUGIN_ROOT, "scripts", "session-lifecycle-hook.mjs"); +const TEST_PLUGIN_DATA = makeTempDir("codex-plugin-test-data-"); + +// Runtime tests exercise real SessionEnd and broker teardown paths. Keep their +// state out of the user's fallback /tmp/codex-companion directory so a test +// session can never discover or clean a live plugin session. +process.env.CLAUDE_PLUGIN_DATA = TEST_PLUGIN_DATA; +test.after(async () => { + const stateRoot = path.join(TEST_PLUGIN_DATA, "state"); + const sessionIds = new Set(); + if (fs.existsSync(stateRoot)) { + for (const entry of fs.readdirSync(stateRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + try { + const session = JSON.parse(fs.readFileSync(path.join(stateRoot, entry.name, "broker.json"), "utf8")); + for (const sessionId of session.sessionIds ?? [session.sessionId]) { + if (sessionId) { + sessionIds.add(sessionId); + } + } + } catch { + // No broker state for this workspace. + } + } + } + for (const sessionId of sessionIds) { + await teardownBrokersForSession(sessionId, { killProcess: terminateProcessTree }); + } + fs.rmSync(TEST_PLUGIN_DATA, { recursive: true, force: true }); +}); async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { const start = Date.now(); @@ -28,6 +65,21 @@ async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { throw new Error("Timed out waiting for condition."); } +function buildSessionEnv(binDir, sessionId) { + return { + ...buildEnv(binDir), + CODEX_COMPANION_SESSION_ID: sessionId + }; +} + +function registerBrokerCleanup(t, cwd, sessionId = null) { + t.after(async () => { + await teardownBrokerForCwd(cwd, sessionId, { + killProcess: terminateProcessTree + }); + }); +} + test("setup reports ready when fake codex is installed and authenticated", () => { const binDir = makeTempDir(); installFakeCodex(binDir); @@ -922,7 +974,7 @@ test("task can finish after subagent work even if the parent turn/completed even assert.equal(result.stdout, "Handled the requested task.\nTask prompt accepted.\n"); }); -test("task using the shared broker still completes when Codex spawns subagents", () => { +test("task using the shared broker still completes when Codex spawns subagents", (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); installFakeCodex(binDir, "with-subagent"); @@ -932,16 +984,16 @@ test("task using the shared broker still completes when Codex spawns subagents", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const sessionId = "sess-shared-subagents"; + const env = buildSessionEnv(binDir, sessionId); + registerBrokerCleanup(t, repo, sessionId); const review = run("node", [SCRIPT, "review"], { cwd: repo, env }); assert.equal(review.status, 0, review.stderr); - if (!loadBrokerSession(repo)) { - return; - } + assert.ok(loadBrokerSession(repo)); const result = run("node", [SCRIPT, "task", "challenge the current design"], { cwd: repo, @@ -1769,7 +1821,7 @@ test("cancel with a job id can still target an active job from another Claude se assert.equal(state.jobs[0].status, "cancelled"); }); -test("cancel sends turn interrupt to the shared app-server before killing a brokered task", async () => { +test("cancel sends turn interrupt to the shared app-server before killing a brokered task", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); @@ -1779,7 +1831,9 @@ test("cancel sends turn interrupt to the shared app-server before killing a brok run("git", ["add", "README.md"], { cwd: repo }); run("git", ["commit", "-m", "init"], { cwd: repo }); - const env = buildEnv(binDir); + const sessionId = "sess-cancel-shared-turn"; + const env = buildSessionEnv(binDir, sessionId); + registerBrokerCleanup(t, repo, sessionId); const launched = run("node", [SCRIPT, "task", "--background", "--json", "investigate the flaky worker timeout"], { cwd: repo, env @@ -1822,15 +1876,6 @@ test("cancel sends turn interrupt to the shared app-server before killing a brok turnId: runningJob.turnId }); - const cleanup = run("node", [SESSION_HOOK, "SessionEnd"], { - cwd: repo, - env, - input: JSON.stringify({ - hook_event_name: "SessionEnd", - cwd: repo - }) - }); - assert.equal(cleanup.status, 0, cleanup.stderr); }); test("session end fully cleans up jobs for the ending session", async (t) => { @@ -2148,7 +2193,7 @@ test("stop hook runs the actual task when auth status looks stale", () => { assert.match(payload.reason, /Missing empty-state guard/i); }); -test("commands lazily start and reuse one shared app-server after first use", async () => { +test("commands without a session owner use a direct app-server", (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); @@ -2161,6 +2206,34 @@ test("commands lazily start and reuse one shared app-server after first use", as fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); const env = buildEnv(binDir); + registerBrokerCleanup(t, repo); + + const review = run("node", [SCRIPT, "review"], { + cwd: repo, + env + }); + assert.equal(review.status, 0, review.stderr); + assert.equal(loadBrokerSession(repo), null); + + const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(fakeState.appServerStarts, 1); +}); + +test("commands lazily start and reuse one shared app-server after first use", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + + const sessionId = "sess-lazy-shared-runtime"; + const env = buildSessionEnv(binDir, sessionId); + registerBrokerCleanup(t, repo, sessionId); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -2169,9 +2242,7 @@ test("commands lazily start and reuse one shared app-server after first use", as assert.equal(review.status, 0, review.stderr); const brokerSession = loadBrokerSession(repo); - if (!brokerSession) { - return; - } + assert.ok(brokerSession); const adversarial = run("node", [SCRIPT, "adversarial-review"], { cwd: repo, @@ -2182,18 +2253,9 @@ test("commands lazily start and reuse one shared app-server after first use", as const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); assert.equal(fakeState.appServerStarts, 1); - const cleanup = run("node", [SESSION_HOOK, "SessionEnd"], { - cwd: repo, - env, - input: JSON.stringify({ - hook_event_name: "SessionEnd", - cwd: repo - }) - }); - assert.equal(cleanup.status, 0, cleanup.stderr); }); -test("setup reuses an existing shared app-server without starting another one", () => { +test("setup reuses an existing shared app-server without starting another one", (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); @@ -2205,7 +2267,9 @@ test("setup reuses an existing shared app-server without starting another one", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const sessionId = "sess-setup-shared-runtime"; + const env = buildSessionEnv(binDir, sessionId); + registerBrokerCleanup(t, repo, sessionId); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -2214,9 +2278,7 @@ test("setup reuses an existing shared app-server without starting another one", assert.equal(review.status, 0, review.stderr); const brokerSession = loadBrokerSession(repo); - if (!brokerSession) { - return; - } + assert.ok(brokerSession); const setup = run("node", [SCRIPT, "setup", "--json"], { cwd: repo, @@ -2227,18 +2289,9 @@ test("setup reuses an existing shared app-server without starting another one", const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); assert.equal(fakeState.appServerStarts, 1); - const cleanup = run("node", [SESSION_HOOK, "SessionEnd"], { - cwd: repo, - env, - input: JSON.stringify({ - hook_event_name: "SessionEnd", - cwd: repo - }) - }); - assert.equal(cleanup.status, 0, cleanup.stderr); }); -test("status reports shared session runtime when a lazy broker is active", () => { +test("status reports shared session runtime when a lazy broker is active", (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); installFakeCodex(binDir); @@ -2248,19 +2301,21 @@ test("status reports shared session runtime when a lazy broker is active", () => run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + const sessionId = "sess-status-shared-runtime"; + const env = buildSessionEnv(binDir, sessionId); + registerBrokerCleanup(t, repo, sessionId); + const review = run("node", [SCRIPT, "review"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(review.status, 0, review.stderr); - if (!loadBrokerSession(repo)) { - return; - } + assert.ok(loadBrokerSession(repo)); const result = run("node", [SCRIPT, "status"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(result.status, 0, result.stderr); From 01c70f57c6a464d90ef9208ab82402a011437c00 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 18:43:46 +0900 Subject: [PATCH 19/20] [COMMON] Track ready broker joiners during cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 브로커 재사용 경쟁 조건 보완: - ready fast-path lock에 세션 소유 정보 게시 - SessionEnd scan이 대기 중 joiner를 marker로 차단 - 동시성 회귀 테스트 추가 --- .../codex/scripts/lib/broker-lifecycle.mjs | 14 ++++-- tests/broker-lifecycle.test.mjs | 43 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ecab46367..ce8bc42bf 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -751,7 +751,10 @@ export async function ensureBrokerSession(cwd, options = {}) { } clearEndedOwnerMarkers(pending.markerFiles); return withOwner; - }, lockOptions); + }, { + ...lockOptions, + ownerSessionId: resolveSessionId(options) + }); if (reused) { return reused; } @@ -935,7 +938,8 @@ export async function teardownBrokersForSession( } const stateExists = fs.existsSync(stateFile); const lockDir = `${stateFile}.lock`; - if (!stateExists && readBrokerLockSession(lockDir) !== sessionId) { + const lockSessionId = readBrokerLockSession(lockDir); + if (!stateExists && lockSessionId !== sessionId) { continue; } @@ -954,7 +958,11 @@ export async function teardownBrokersForSession( preview = null; } } - if (preview && !brokerSessionOwners(preview).includes(sessionId)) { + if ( + preview && + !brokerSessionOwners(preview).includes(sessionId) && + lockSessionId !== sessionId + ) { continue; } try { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index e4b47bb9c..039cc93e2 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -1300,6 +1300,49 @@ test("mismatched-cwd SessionEnd marks an in-flight broker spawn before broker.js }); }); +test("session-wide teardown marks a session joining a ready broker under its lock", async () => { + await withPluginData(async () => { + await withReadyBroker(async ({ endpoint, requests, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, + pidFile: null, + logFile: null, + sessionDir, + pid: null, + sessionId: "A", + sessionIds: ["A"] + }); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); + const lockDir = `${stateFile}.lock`; + fs.mkdirSync(lockDir); + fs.writeFileSync(path.join(lockDir, "owner"), `${process.pid}-ready-reuse`, "utf8"); + fs.writeFileSync(path.join(lockDir, "session"), "S", "utf8"); + + try { + await assert.rejects( + () => teardownBrokersForSession("S", { + killProcess: () => {}, + lockTimeoutMs: 50 + }), + { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } + ); + } finally { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + + await assert.rejects( + () => ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "S" } + }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["A"]); + assert.equal(requests.length, 0); + }); + }); +}); + test("CodexAppServerClient does not fall back to a direct app-server after its broker owner ends", async () => { await withPluginData(async () => { const cwd = makeTempDir(); From 937fe5d0809c06815fe27df594c372e06f1f9862 Mon Sep 17 00:00:00 2001 From: Hongsu Ryu Date: Mon, 20 Jul 2026 20:00:06 +0900 Subject: [PATCH 20/20] [COMMON] Make ended broker sessions monotonic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 세션 종료와 broker admission의 경쟁 조건을 구조적으로 차단: - 종료 상태를 전역 단조 tombstone으로 게시해 다른 세션이 소비하지 않도록 변경 - broker, 제공 endpoint, direct fallback의 연결 전후에 종료 세션 admission 검사 - lock 대기와 초기화 경계를 재현하는 결정적 회귀 테스트 추가 --- plugins/codex/scripts/lib/app-server.mjs | 24 +- .../codex/scripts/lib/broker-lifecycle.mjs | 303 +++++++----------- .../codex/scripts/session-lifecycle-hook.mjs | 10 +- tests/broker-lifecycle.test.mjs | 164 +++++++--- 4 files changed, 267 insertions(+), 234 deletions(-) diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 6c9e141ca..2e2b43bee 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -13,7 +13,12 @@ import process from "node:process"; import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { ensureBrokerSession, resolveSessionId, reuseBrokerSession } from "./broker-lifecycle.mjs"; +import { + assertBrokerSessionActive, + ensureBrokerSession, + resolveSessionId, + reuseBrokerSession +} from "./broker-lifecycle.mjs"; import { terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); @@ -334,10 +339,17 @@ class BrokerCodexAppServerClient extends AppServerClientBase { export class CodexAppServerClient { static async connect(cwd, options = {}) { + const sessionId = resolveSessionId({ env: options.env }); let brokerEndpoint = null; - if (!options.disableBroker) { + if (options.disableBroker) { + assertBrokerSessionActive(sessionId); + } else { brokerEndpoint = options.brokerEndpoint ?? options.env?.[BROKER_ENDPOINT_ENV] ?? process.env[BROKER_ENDPOINT_ENV] ?? null; - const sessionId = resolveSessionId({ env: options.env }); + if (brokerEndpoint) { + // A supplied endpoint bypasses the lifecycle functions that normally + // perform admission under the broker-state lock. + assertBrokerSessionActive(sessionId); + } if (!brokerEndpoint && sessionId && options.reuseExistingBroker) { const brokerSession = await reuseBrokerSession(cwd, { ...options.brokerOptions, @@ -359,6 +371,12 @@ export class CodexAppServerClient { ? new BrokerCodexAppServerClient(cwd, { ...options, brokerEndpoint }) : new SpawnedCodexAppServerClient(cwd, options); await client.initialize(); + try { + assertBrokerSessionActive(sessionId); + } catch (error) { + await client.close(); + throw error; + } return client; } } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ce8bc42bf..3388f4cec 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -19,7 +19,7 @@ const BROKER_LOCK_TIMEOUT_CODE = "EBROKERSTATELOCKTIMEOUT"; export const BROKER_OWNER_ENDED_CODE = "EBROKEROWNERENDED"; export const BROKER_CLEANUP_INCOMPLETE_CODE = "EBROKERCLEANUPINCOMPLETE"; const MAX_BROKER_STATE_BYTES = 64 * 1024; -const ENDED_OWNER_MARKER_PREFIX = `${BROKER_STATE_FILE}.ended-`; +const ENDED_SESSIONS_DIR = ".ended-sessions"; const BROKER_LOCK_SESSION_FILE = "session"; let brokerLockTokenSeq = 0; @@ -479,72 +479,90 @@ export function loadBrokerSession(cwd) { return readBrokerStateFile(resolveBrokerStateFile(cwd)); } -function endedOwnerMarkerFile(stateFile, sessionId) { +function endedSessionFile(sessionId) { const digest = createHash("sha256").update(sessionId).digest("hex"); - return path.join(path.dirname(stateFile), `${ENDED_OWNER_MARKER_PREFIX}${digest}`); + return path.join(resolveStateRoot(), ENDED_SESSIONS_DIR, digest); } -function recordEndedBrokerOwner(stateFile, sessionId) { +export function markBrokerSessionEnded(sessionId) { if (!sessionId || Buffer.byteLength(sessionId, "utf8") > 1024) { - return; + return false; } - fs.mkdirSync(path.dirname(stateFile), { recursive: true }); + const markerFile = endedSessionFile(sessionId); + fs.mkdirSync(path.dirname(markerFile), { recursive: true, mode: 0o700 }); + const candidate = `${markerFile}.tmp-${process.pid}-${brokerLockTokenSeq += 1}`; try { - fs.writeFileSync(endedOwnerMarkerFile(stateFile, sessionId), sessionId, { encoding: "utf8", flag: "wx" }); - } catch (error) { - if (error?.code !== "EEXIST") { - throw error; - } + // Publish only a complete value. A direct `wx` write makes the final path + // visible before all bytes are present, allowing another process to + // transiently treat an ending session as active. + fs.writeFileSync(candidate, sessionId, { encoding: "utf8", flag: "wx", mode: 0o600 }); + fs.renameSync(candidate, markerFile); + } finally { + fs.rmSync(candidate, { force: true }); } + return true; } -function applyEndedBrokerOwners(stateFile, session) { - const markerFiles = []; - const endedOwners = new Set(); - for (const entry of fs.readdirSync(path.dirname(stateFile), { withFileTypes: true })) { - if (!entry.isFile() || !entry.name.startsWith(ENDED_OWNER_MARKER_PREFIX)) { - continue; +export function isBrokerSessionEnded(sessionId) { + if (!sessionId || Buffer.byteLength(sessionId, "utf8") > 1024) { + return false; + } + const markerFile = endedSessionFile(sessionId); + let descriptor = null; + try { + const before = fs.lstatSync(markerFile); + if (!before.isFile() || before.size > 1024) { + throw Object.assign(new Error(`Invalid ended-session marker: ${markerFile}`), { + code: "EINVALIDENDEDSESSIONMARKER" + }); } - const markerFile = path.join(path.dirname(stateFile), entry.name); - let descriptor = null; - try { - const before = fs.lstatSync(markerFile); - if (!before.isFile() || before.size > 1024) { - continue; - } - const noFollow = fs.constants.O_NOFOLLOW ?? 0; - const nonBlock = fs.constants.O_NONBLOCK ?? 0; - descriptor = fs.openSync(markerFile, fs.constants.O_RDONLY | noFollow | nonBlock); - const opened = fs.fstatSync(descriptor); - if (!opened.isFile() || opened.size > 1024 || - opened.dev !== before.dev || opened.ino !== before.ino) { - continue; - } - const owner = readBoundedUtf8(descriptor, 1024); - const expectedName = owner - ? `${ENDED_OWNER_MARKER_PREFIX}${createHash("sha256").update(owner).digest("hex")}` - : null; - if (owner && entry.name === expectedName) { - endedOwners.add(owner); - markerFiles.push(markerFile); - } - } catch { - // Leave unreadable markers for a later cleanup attempt. - } finally { - if (descriptor != null) { - try { - fs.closeSync(descriptor); - } catch { - // Ignore a descriptor already closed by a concurrent test shim. - } + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + descriptor = fs.openSync(markerFile, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.size > 1024 || + opened.dev !== before.dev || opened.ino !== before.ino) { + throw Object.assign(new Error(`Changed ended-session marker: ${markerFile}`), { + code: "EINVALIDENDEDSESSIONMARKER" + }); + } + if (readBoundedUtf8(descriptor, 1024) !== sessionId) { + throw Object.assign(new Error(`Mismatched ended-session marker: ${markerFile}`), { + code: "EINVALIDENDEDSESSIONMARKER" + }); + } + return true; + } catch (error) { + if (error?.code === "ENOENT") { + return false; + } + throw error; + } finally { + if (descriptor != null) { + try { + fs.closeSync(descriptor); + } catch { + // Ignore a descriptor already closed by a concurrent test shim. } } } +} + +export function assertBrokerSessionActive(sessionId) { + if (sessionId && isBrokerSessionEnded(sessionId)) { + throw brokerOwnerEndedError(sessionId); + } +} + +function applyEndedBrokerOwners(session) { + const endedOwners = new Set( + brokerSessionOwners(session).filter((owner) => isBrokerSessionEnded(owner)) + ); if (endedOwners.size === 0) { - return { session, markerFiles, endedOwners }; + return { session, endedOwners }; } if (!session) { - return { session, markerFiles, endedOwners }; + return { session, endedOwners }; } const owners = brokerSessionOwners(session).filter((owner) => !endedOwners.has(owner)); return { @@ -553,21 +571,10 @@ function applyEndedBrokerOwners(stateFile, session) { sessionId: owners[0] ?? null, sessionIds: owners }, - markerFiles, endedOwners }; } -function clearEndedOwnerMarkers(markerFiles) { - for (const markerFile of markerFiles) { - try { - fs.unlinkSync(markerFile); - } catch { - // A later locked pass can safely retry stale marker cleanup. - } - } -} - // Write broker.json atomically (temp + rename) so a concurrent reader — e.g. an // unlocked ownership pre-check in another session's teardown — never observes a // half-written file and mis-parses it. @@ -664,7 +671,7 @@ function discardBrokerSession(cwd, session, options) { } function rejectEndedBrokerOwner(cwd, pending, sessionId, options) { - if (!sessionId || !pending.endedOwners.has(sessionId)) { + if (!sessionId || (!pending.endedOwners.has(sessionId) && !isBrokerSessionEnded(sessionId))) { return; } const remainingOwners = brokerSessionOwners(pending.session); @@ -673,7 +680,6 @@ function rejectEndedBrokerOwner(cwd, pending, sessionId, options) { } else { discardBrokerSession(cwd, pending.session, options); } - clearEndedOwnerMarkers(pending.markerFiles); throw brokerOwnerEndedError(sessionId); } @@ -681,33 +687,31 @@ function rejectEndedBrokerOwner(cwd, pending, sessionId, options) { // and persist a fresh one. Callers run this inside the state-file lock so two // racing sessions cannot each leave an orphaned broker with no broker.json. async function adoptOrSpawnBroker(cwd, options) { - const stateFile = resolveBrokerStateFile(cwd); const sessionId = resolveSessionId(options); - const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + const pending = applyEndedBrokerOwners(loadBrokerSession(cwd)); rejectEndedBrokerOwner(cwd, pending, sessionId, options); const current = pending.session; if (current && (await isBrokerEndpointReady(current.endpoint))) { - const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); - if (withOwner !== current || pending.markerFiles.length > 0) { + const pendingAfterProbe = applyEndedBrokerOwners(current); + rejectEndedBrokerOwner(cwd, pendingAfterProbe, sessionId, options); + const active = pendingAfterProbe.session; + const withOwner = withBrokerSessionOwner(active, sessionId); + if (withOwner !== current) { saveBrokerSession(cwd, withOwner); } - clearEndedOwnerMarkers(pending.markerFiles); return withOwner; } discardBrokerSession(cwd, current, options); - clearEndedOwnerMarkers(pending.markerFiles); const session = await spawnReadyBroker(cwd, options); if (!session) { - const markerFile = sessionId ? endedOwnerMarkerFile(stateFile, sessionId) : null; - if (markerFile && fs.existsSync(markerFile)) { - clearEndedOwnerMarkers([markerFile]); + if (isBrokerSessionEnded(sessionId)) { throw brokerOwnerEndedError(sessionId); } return null; } const spawnedWithOwner = withBrokerSessionOwner(session, session.sessionId); - const pendingAfterSpawn = applyEndedBrokerOwners(stateFile, spawnedWithOwner); + const pendingAfterSpawn = applyEndedBrokerOwners(spawnedWithOwner); const activeOwners = brokerSessionOwners(pendingAfterSpawn.session); if (activeOwners.length === 0) { teardownBrokerSession({ @@ -718,16 +722,15 @@ async function adoptOrSpawnBroker(cwd, options) { pid: session.pid, killProcess: options.killProcess ?? null }); - clearEndedOwnerMarkers(pendingAfterSpawn.markerFiles); throw brokerOwnerEndedError(session.sessionId); } saveBrokerSession(cwd, pendingAfterSpawn.session); - clearEndedOwnerMarkers(pendingAfterSpawn.markerFiles); return pendingAfterSpawn.session; } export async function ensureBrokerSession(cwd, options = {}) { const stateFile = resolveBrokerStateFile(cwd); + const sessionId = resolveSessionId(options); const lockOptions = options.lockTimeoutMs == null ? {} : { timeoutMs: options.lockTimeoutMs }; // Fast path: reuse a ready broker. The readiness probe runs outside the lock, @@ -739,21 +742,20 @@ export async function ensureBrokerSession(cwd, options = {}) { break; } const reused = await withBrokerStateFileLock(stateFile, () => { - const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); - rejectEndedBrokerOwner(cwd, pending, resolveSessionId(options), options); + const pending = applyEndedBrokerOwners(loadBrokerSession(cwd)); + rejectEndedBrokerOwner(cwd, pending, sessionId, options); const current = pending.session; if (!current || current.endpoint !== existing.endpoint) { return null; } - const withOwner = withBrokerSessionOwner(current, resolveSessionId(options)); - if (withOwner !== current || pending.markerFiles.length > 0) { + const withOwner = withBrokerSessionOwner(current, sessionId); + if (withOwner !== current) { saveBrokerSession(cwd, withOwner); } - clearEndedOwnerMarkers(pending.markerFiles); return withOwner; }, { ...lockOptions, - ownerSessionId: resolveSessionId(options) + ownerSessionId: sessionId }); if (reused) { return reused; @@ -767,42 +769,42 @@ export async function ensureBrokerSession(cwd, options = {}) { fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); return withBrokerStateFileLock(stateFile, () => adoptOrSpawnBroker(cwd, options), { ...lockOptions, - ownerSessionId: resolveSessionId(options) + ownerSessionId: sessionId }); } // Reuse cwd's recorded broker without claiming ownership or spawning a new -// broker. Probe-only callers still have to honor ended-owner markers; reading -// broker.json directly can otherwise let an ended session reconnect after its -// teardown was deferred by lock contention. +// broker. Probe-only callers still have to honor the global ended-session +// tombstone; reading broker.json directly can otherwise let an ended session +// reconnect after its teardown was deferred by lock contention. export async function reuseBrokerSession(cwd, options = {}) { const stateFile = resolveBrokerStateFile(cwd); + const sessionId = resolveSessionId(options); // Always take the state lock, even when broker.json has not been published. - // SessionEnd can intentionally leave only an ended-owner marker when it - // races before the first broker lock; probe-only reuse must reconcile that - // marker before it is allowed to fall back to a direct app-server. + // SessionEnd can intentionally leave only a global tombstone when it races + // before the first broker lock; probe-only reuse must reconcile that status + // before it is allowed to fall back to a direct app-server. fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); const lockOptions = options.lockTimeoutMs == null ? {} : { timeoutMs: options.lockTimeoutMs }; return withBrokerStateFileLock(stateFile, async () => { - const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); - rejectEndedBrokerOwner(cwd, pending, resolveSessionId(options), options); + const pending = applyEndedBrokerOwners(loadBrokerSession(cwd)); + rejectEndedBrokerOwner(cwd, pending, sessionId, options); const current = pending.session; if (!current || brokerSessionOwners(current).length === 0) { discardBrokerSession(cwd, current, options); - clearEndedOwnerMarkers(pending.markerFiles); return null; } if (!(await isBrokerEndpointReady(current.endpoint))) { discardBrokerSession(cwd, current, options); - clearEndedOwnerMarkers(pending.markerFiles); return null; } - if (pending.markerFiles.length > 0) { - saveBrokerSession(cwd, current); + const pendingAfterProbe = applyEndedBrokerOwners(current); + rejectEndedBrokerOwner(cwd, pendingAfterProbe, sessionId, options); + if (pendingAfterProbe.session !== current) { + saveBrokerSession(cwd, pendingAfterProbe.session); } - clearEndedOwnerMarkers(pending.markerFiles); - return current; + return pendingAfterProbe.session; }, lockOptions); } @@ -818,49 +820,28 @@ export async function teardownBrokerForCwd( ) { const stateFile = resolveBrokerStateFile(cwd); const lockDir = `${stateFile}.lock`; + markBrokerSessionEnded(sessionId); if (!fs.existsSync(stateFile) && !fs.existsSync(lockDir) && !fallbackSession) { - if (sessionId) { - // Persist the end intent even before a concurrent first broker start has - // published its lock. The starter will reconcile this marker before it - // can spawn or persist ownership for the ended session. - recordEndedBrokerOwner(stateFile, sessionId); - } return false; } fs.mkdirSync(resolveStateDir(cwd), { recursive: true }); - // Publish the end intent before waiting for the state lock. A lock holder - // that completes while this hook is waiting can then reconcile the owner, - // and a timeout cannot leave a tombstone that arrived after the final pass. - recordEndedBrokerOwner(stateFile, sessionId); try { return await withBrokerStateFileLock( stateFile, async () => { - const pending = applyEndedBrokerOwners(stateFile, loadBrokerSession(cwd)); + const recorded = loadBrokerSession(cwd); + const pending = applyEndedBrokerOwners(recorded); const current = pending.session; const owners = brokerSessionOwners(current); - if (sessionId && owners.length > 0) { - if (!owners.includes(sessionId)) { - if (pending.markerFiles.length > 0) { - writeBrokerStateFile(stateFile, current); - clearEndedOwnerMarkers(pending.markerFiles); - } - return false; - } - const remainingOwners = owners.filter((owner) => owner !== sessionId); - if (remainingOwners.length > 0) { - writeBrokerStateFile(stateFile, { - ...current, - sessionId: remainingOwners[0], - sessionIds: remainingOwners - }); - clearEndedOwnerMarkers(pending.markerFiles); - return false; + if (current && owners.length > 0) { + if (current !== recorded) { + writeBrokerStateFile(stateFile, current); } + return false; } - const session = current ?? fallbackSession; + const session = recorded ?? fallbackSession; if (session?.endpoint) { await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownTimeoutMs }); } @@ -875,7 +856,6 @@ export async function teardownBrokerForCwd( if (fs.existsSync(stateFile)) { fs.unlinkSync(stateFile); } - clearEndedOwnerMarkers(pending.markerFiles); return true; }, { timeoutMs: lockTimeoutMs } @@ -901,6 +881,7 @@ export async function teardownBrokersForSession( if (!sessionId) { return 0; } + markBrokerSessionEnded(sessionId); const stateRoot = resolveStateRoot(); if (!fs.existsSync(stateRoot)) { return 0; @@ -965,13 +946,6 @@ export async function teardownBrokersForSession( ) { continue; } - try { - recordEndedBrokerOwner(stateFile, sessionId); - } catch (error) { - teardownError ??= error; - continue; - } - const remaining = deadline != null ? deadline - Date.now() : lockTimeoutMs; if (remaining <= 0) { incompleteReason = "deadline"; @@ -983,77 +957,42 @@ export async function teardownBrokersForSession( await withBrokerStateFileLock( stateFile, async () => { - const pending = applyEndedBrokerOwners(stateFile, readBrokerStateFile(stateFile)); - const session = pending.session; - if (!session) { - clearEndedOwnerMarkers(pending.markerFiles); + const recorded = readBrokerStateFile(stateFile); + if (!recorded) { return; } + const session = applyEndedBrokerOwners(recorded).session; const owners = brokerSessionOwners(session); - if (!owners.includes(sessionId)) { - if (pending.markerFiles.length > 0) { - if (owners.length === 0) { - if (session.endpoint) { - const shutdownWait = - deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; - if (shutdownWait > 0) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); - } - } - teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, - killProcess - }); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); - } - count += 1; - } else { - writeBrokerStateFile(stateFile, session); - } - clearEndedOwnerMarkers(pending.markerFiles); + if (owners.length > 0) { + if (session !== recorded) { + writeBrokerStateFile(stateFile, session); } return; } - const remainingOwners = owners.filter((owner) => owner !== sessionId); - if (remainingOwners.length > 0) { - writeBrokerStateFile(stateFile, { - ...session, - sessionId: remainingOwners[0], - sessionIds: remainingOwners - }); - clearEndedOwnerMarkers(pending.markerFiles); - return; - } // Bound the graceful-shutdown wait by the remaining scan budget, not // just the per-RPC default: several unresponsive endpoints could // otherwise each burn shutdownTimeoutMs and push the whole scan past // the SessionEnd hook's budget. Out of budget → skip the RPC and let // teardownBrokerSession terminate the process directly. - if (session.endpoint) { + if (recorded.endpoint) { const shutdownWait = deadline != null ? Math.min(shutdownTimeoutMs, deadline - Date.now()) : shutdownTimeoutMs; if (shutdownWait > 0) { - await sendBrokerShutdown(session.endpoint, { timeoutMs: shutdownWait }); + await sendBrokerShutdown(recorded.endpoint, { timeoutMs: shutdownWait }); } } teardownBrokerSession({ - endpoint: session.endpoint ?? null, - pidFile: session.pidFile ?? null, - logFile: session.logFile ?? null, - sessionDir: session.sessionDir ?? null, - pid: session.pid ?? null, + endpoint: recorded.endpoint ?? null, + pidFile: recorded.pidFile ?? null, + logFile: recorded.logFile ?? null, + sessionDir: recorded.sessionDir ?? null, + pid: recorded.pid ?? null, killProcess }); if (fs.existsSync(stateFile)) { fs.unlinkSync(stateFile); } - clearEndedOwnerMarkers(pending.markerFiles); count += 1; }, { timeoutMs: entryLockTimeout } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index cf0dc9156..a3c116c21 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -11,6 +11,7 @@ import { BROKER_CLEANUP_INCOMPLETE_CODE, LOG_FILE_ENV, PID_FILE_ENV, + markBrokerSessionEnded, teardownBrokerForCwd, teardownBrokersForSession } from "./lib/broker-lifecycle.mjs"; @@ -218,15 +219,20 @@ export async function handleSessionEnd(input) { const sessionId = input.session_id || process.env[SESSION_ID_ENV]; const cleanupDeadline = Date.now() + SESSION_END_CLEANUP_BUDGET_MS; let cleanupError = null; + try { + markBrokerSessionEnded(sessionId); + } catch (error) { + cleanupError = error; + } let jobDiscoveryComplete = true; let cwdBrokerTeardownSafe = true; try { const cleanup = cleanupSessionJobs(cwd, sessionId, cleanupDeadline); - cleanupError = cleanup.error; + cleanupError ??= cleanup.error; jobDiscoveryComplete = cleanup.discoveryComplete; cwdBrokerTeardownSafe = cleanup.cwdBrokerTeardownSafe; } catch (error) { - cleanupError = error; + cleanupError ??= error; jobDiscoveryComplete = false; cwdBrokerTeardownSafe = false; } diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs index 039cc93e2..7e16490fd 100644 --- a/tests/broker-lifecycle.test.mjs +++ b/tests/broker-lifecycle.test.mjs @@ -23,6 +23,7 @@ import { BROKER_CLEANUP_INCOMPLETE_CODE, BROKER_OWNER_ENDED_CODE, ensureBrokerSession, + isBrokerSessionEnded, loadBrokerSession, resolveSessionId, saveBrokerSession, @@ -698,6 +699,7 @@ test("handleSessionEnd reports incomplete cleanup when job-state discovery is in assert.equal(loadState(jobWorkspace).jobs.length, 1); assert.notEqual(loadBrokerSession(jobWorkspace), null); + assert.equal(isBrokerSessionEnded("S"), true); }); }); @@ -1384,9 +1386,93 @@ test("CodexAppServerClient does not fall back to a direct app-server after its b }); }); +test("CodexAppServerClient rejects an ended session before using a supplied broker endpoint", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + await teardownBrokerForCwd(cwd, "S", { killProcess: () => {} }); + + await assert.rejects( + () => CodexAppServerClient.connect(cwd, { + brokerEndpoint: createBrokerEndpoint(path.join(makeTempDir(), "broker.sock")), + env: { CODEX_COMPANION_SESSION_ID: "S" } + }), + { code: BROKER_OWNER_ENDED_CODE } + ); + }); +}); + +test("CodexAppServerClient rejects a supplied endpoint when its session ends during initialize", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const sessionDir = makeTempDir(); + const endpoint = createBrokerEndpoint(sessionDir); + const target = parseBrokerEndpoint(endpoint); + let releaseInitialize; + const initializeReceived = new Promise((resolve) => { + releaseInitialize = resolve; + }); + let socket = null; + const server = net.createServer((connected) => { + socket = connected; + connected.once("data", () => releaseInitialize()); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(target.path, () => { + server.off("error", reject); + resolve(); + }); + }); + + try { + const pending = CodexAppServerClient.connect(cwd, { + brokerEndpoint: endpoint, + env: { CODEX_COMPANION_SESSION_ID: "S" } + }); + await initializeReceived; + await teardownBrokerForCwd(cwd, "S", { killProcess: () => {} }); + socket.write(`${JSON.stringify({ id: 1, result: {} })}\n`); + + await assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); + } finally { + socket?.destroy(); + await new Promise((resolve) => server.close(resolve)); + } + }); +}); + +test("CodexAppServerClient rejects an ended session before direct fallback", async () => { + await withPluginData(async () => { + const cwd = makeTempDir(); + const directMarker = path.join(makeTempDir(), "direct-spawned"); + const fakeBin = makeTempDir(); + const fakeCodex = path.join(fakeBin, "codex"); + fs.writeFileSync( + fakeCodex, + `#!/bin/sh\nprintf spawned > "${directMarker}"\nexit 1\n`, + { encoding: "utf8", mode: 0o755 } + ); + await teardownBrokerForCwd(cwd, "S", { killProcess: () => {} }); + + await assert.rejects( + () => CodexAppServerClient.connect(cwd, { + disableBroker: true, + env: { + ...process.env, + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + CODEX_COMPANION_SESSION_ID: "S" + } + }), + { code: BROKER_OWNER_ENDED_CODE } + ); + assert.equal(fs.existsSync(directMarker), false); + }); +}); + test("CodexAppServerClient does not fall back after an ended owner's broker spawn times out", async () => { await withPluginData(async () => { const cwd = makeTempDir(); + const stateFile = path.join(resolveStateDir(cwd), "broker.json"); const spawnMarker = path.join(makeTempDir(), "broker-spawned"); const directMarker = path.join(makeTempDir(), "direct-spawned"); const fakeBin = makeTempDir(); @@ -1405,15 +1491,16 @@ test("CodexAppServerClient does not fall back after an ended owner's broker spaw TEST_BROKER_SPAWN_MARKER: spawnMarker, TEST_BROKER_LISTEN_DELAY_MS: "1000" }, - brokerOptions: { scriptPath: writeFakeBrokerScript(), timeoutMs: 100 } + brokerOptions: { scriptPath: writeFakeBrokerScript(), timeoutMs: 2000 } }); const rejected = assert.rejects(pending, { code: BROKER_OWNER_ENDED_CODE }); const deadline = Date.now() + 2000; - while (!fs.existsSync(spawnMarker) && Date.now() < deadline) { + while ((!fs.existsSync(spawnMarker) || !fs.existsSync(`${stateFile}.lock`)) && Date.now() < deadline) { await sleep(10); } assert.equal(fs.existsSync(spawnMarker), true); + assert.equal(fs.existsSync(`${stateFile}.lock`), true); await assert.rejects( () => teardownBrokerForCwd(cwd, "S", { killProcess: () => {}, lockTimeoutMs: 50 }), { code: BROKER_CLEANUP_INCOMPLETE_CODE, reason: "lock-timeout" } @@ -1576,15 +1663,14 @@ test("sendBrokerShutdown returns immediately for a non-positive timeout", async }); }); -test("teardownBrokerForCwd records an ended owner before the first broker lock exists", async () => { +test("teardownBrokerForCwd records a global ended session before the first broker lock exists", async () => { await withPluginData(async () => { const cwd = makeTempDir(); - const stateDir = resolveStateDir(cwd); - const tornDown = await teardownBrokerForCwd(cwd, "S"); assert.equal(tornDown, false); - assert.equal(fs.existsSync(stateDir), true); + assert.equal(isBrokerSessionEnded("S"), true); + assert.equal(fs.existsSync(resolveStateDir(cwd)), false); await assert.rejects( () => ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "S" } }), { code: BROKER_OWNER_ENDED_CODE } @@ -1693,21 +1779,20 @@ test("teardownBrokersForSession continues after one broker cleanup fails", async }); }); -test("teardownBrokersForSession continues after one ended-owner marker write fails", async () => { +test("teardownBrokersForSession does not mutate brokers when the global end marker cannot be published", async () => { await withPluginData(async () => { const stateRoot = stateRootForTest(); - const firstJson = writeBrokerJson(stateRoot, "worktree-0000markerfail", { + const firstJson = writeBrokerJson(stateRoot, "worktree-0000first", { endpoint: "unix:/tmp/codex-test-marker-failure.sock", sessionId: "S" }); - const secondJson = writeBrokerJson(stateRoot, "worktree-9999markersuccess", { + const secondJson = writeBrokerJson(stateRoot, "worktree-9999second", { endpoint: "unix:/tmp/codex-test-marker-success.sock", sessionId: "S" }); - const firstDir = path.dirname(firstJson); const originalWriteFileSync = fs.writeFileSync.bind(fs); fs.writeFileSync = (file, ...args) => { - if (path.dirname(String(file)) === firstDir && path.basename(String(file)).startsWith("broker.json.ended-")) { + if (path.dirname(String(file)) === path.join(stateRoot, ".ended-sessions")) { throw Object.assign(new Error("simulated marker write failure"), { code: "EACCES" }); } return originalWriteFileSync(file, ...args); @@ -1723,49 +1808,34 @@ test("teardownBrokersForSession continues after one ended-owner marker write fai } assert.equal(fs.existsSync(firstJson), true); - assert.equal(fs.existsSync(secondJson), false); + assert.equal(fs.existsSync(secondJson), true); }); }); -test("teardownBrokersForSession preserves incomplete status when later cleanup also fails", async () => { +test("an unrelated broker reuse cannot consume another session's ended status", async () => { await withPluginData(async () => { - const stateRoot = stateRootForTest(); - const lockedJson = writeBrokerJson(stateRoot, "worktree-0000locked", { - endpoint: "unix:/tmp/codex-test-locked-first.sock", - sessionId: "S" - }); - const failedJson = writeBrokerJson(stateRoot, "worktree-9999markerfail", { - endpoint: "unix:/tmp/codex-test-marker-failure-later.sock", - sessionId: "S" - }); - const failedDir = path.dirname(failedJson); - const lockDir = `${lockedJson}.lock`; - fs.mkdirSync(lockDir); - const originalWriteFileSync = fs.writeFileSync.bind(fs); - fs.writeFileSync = (file, ...args) => { - if (path.dirname(String(file)) === failedDir && path.basename(String(file)).startsWith("broker.json.ended-")) { - throw Object.assign(new Error("simulated later marker failure"), { code: "EACCES" }); - } - return originalWriteFileSync(file, ...args); - }; + await withReadyBroker(async ({ endpoint, sessionDir }) => { + const cwd = makeTempDir(); + saveBrokerSession(cwd, { + endpoint, pidFile: null, logFile: null, sessionDir, pid: null, + sessionId: "A", sessionIds: ["A"] + }); + + assert.equal(await teardownBrokerForCwd(makeTempDir(), "S"), false); + assert.equal(isBrokerSessionEnded("S"), true); + + const reused = await ensureBrokerSession(cwd, { + env: { CODEX_COMPANION_SESSION_ID: "C" } + }); + assert.deepEqual(reused.sessionIds, ["A", "C"]); + assert.equal(isBrokerSessionEnded("S"), true); - try { await assert.rejects( - () => teardownBrokersForSession("S", { killProcess: () => {}, lockTimeoutMs: 50 }), - (error) => { - assert.equal(error.code, BROKER_CLEANUP_INCOMPLETE_CODE); - assert.equal(error.reason, "lock-timeout"); - assert.equal(error.cause?.code, "EACCES"); - return true; - } + () => ensureBrokerSession(cwd, { env: { CODEX_COMPANION_SESSION_ID: "S" } }), + { code: BROKER_OWNER_ENDED_CODE } ); - } finally { - fs.writeFileSync = originalWriteFileSync; - fs.rmSync(lockDir, { recursive: true, force: true }); - } - - assert.equal(fs.existsSync(lockedJson), true); - assert.equal(fs.existsSync(failedJson), true); + assert.deepEqual(loadBrokerSession(cwd).sessionIds, ["A", "C"]); + }); }); });