diff --git a/plugins/codex/hooks/hooks.json b/plugins/codex/hooks/hooks.json index 19e33b818..1917ac986 100644 --- a/plugins/codex/hooks/hooks.json +++ b/plugins/codex/hooks/hooks.json @@ -18,7 +18,7 @@ { "type": "command", "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/session-lifecycle-hook.mjs\" SessionEnd", - "timeout": 5 + "timeout": 30 } ] } diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 1954274fe..b55c14ff7 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -1,6 +1,5 @@ #!/usr/bin/env node -import fs from "node:fs"; import net from "node:net"; import path from "node:path"; import process from "node:process"; @@ -8,6 +7,7 @@ import process from "node:process"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; +import { ensurePrivateDir, removeFileIfExists, writePrivateFile } from "./lib/fs.mjs"; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); @@ -41,28 +41,34 @@ function writePidFile(pidFile) { if (!pidFile) { return; } - fs.mkdirSync(path.dirname(pidFile), { recursive: true }); - fs.writeFileSync(pidFile, `${process.pid}\n`, "utf8"); + ensurePrivateDir(path.dirname(pidFile)); + writePrivateFile(pidFile, `${process.pid}\n`); } async function main() { const [subcommand, ...argv] = process.argv.slice(2); if (subcommand !== "serve") { - throw new Error("Usage: node scripts/app-server-broker.mjs serve --endpoint [--cwd ] [--pid-file ]"); + throw new Error( + "Usage: node scripts/app-server-broker.mjs serve --endpoint --instance-token [--cwd ] [--pid-file ]" + ); } const { options } = parseArgs(argv, { - valueOptions: ["cwd", "pid-file", "endpoint"] + valueOptions: ["cwd", "pid-file", "endpoint", "instance-token"] }); if (!options.endpoint) { throw new Error("Missing required --endpoint."); } + if (!options["instance-token"]) { + throw new Error("Missing required --instance-token."); + } const cwd = options.cwd ? path.resolve(process.cwd(), options.cwd) : process.cwd(); const endpoint = String(options.endpoint); const listenTarget = parseBrokerEndpoint(endpoint); const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null; + const instanceToken = String(options["instance-token"]); writePidFile(pidFile); const appClient = await CodexAppServerClient.connect(cwd, { disableBroker: true }); @@ -99,18 +105,20 @@ async function main() { } } - async function shutdown(server) { + async function shutdown(server, responseSocket = null) { for (const socket of sockets) { - socket.end(); + if (socket === responseSocket) { + socket.destroySoon(); + } else { + socket.destroy(); + } } await appClient.close().catch(() => {}); await new Promise((resolve) => server.close(resolve)); - if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { - fs.unlinkSync(listenTarget.path); - } - if (pidFile && fs.existsSync(pidFile)) { - fs.unlinkSync(pidFile); + if (listenTarget.kind === "unix") { + removeFileIfExists(listenTarget.path); } + removeFileIfExists(pidFile); } appClient.setNotificationHandler(routeNotification); @@ -158,8 +166,18 @@ async function main() { } if (message.id !== undefined && message.method === "broker/shutdown") { - send(socket, { id: message.id, result: {} }); - await shutdown(server); + if (message.params?.instanceToken !== instanceToken) { + send(socket, { + id: message.id, + error: buildJsonRpcError(-32003, "Broker shutdown identity did not match this instance.") + }); + continue; + } + send(socket, { + id: message.id, + result: { pid: process.pid, instanceToken } + }); + await shutdown(server, socket); process.exit(0); } @@ -233,15 +251,12 @@ async function main() { }); }); - process.on("SIGTERM", async () => { - await shutdown(server); - process.exit(0); - }); - - process.on("SIGINT", async () => { - await shutdown(server); - process.exit(0); - }); + for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, async () => { + await shutdown(server); + process.exit(0); + }); + } server.listen(listenTarget.path); } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ef763819c..978509598 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import fs from "node:fs"; import net from "node:net"; import os from "node:os"; @@ -5,15 +6,35 @@ import path from "node:path"; import process from "node:process"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; + import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; +import { + ensurePrivateDir, + PRIVATE_DIR_MODE, + PRIVATE_FILE_MODE, + removeFileIfExists, + setMode, + writeJsonFileAtomic +} from "./fs.mjs"; +import { withLock } from "./locking.mjs"; +import { + getProcessIdentity, + isProcessRunning, + isProcessTreeRunning, + isValidPid, + processHasLaunchSequence, + processHasLaunchToken, + terminateProcessTree, + waitForProcessExit +} from "./process.mjs"; import { resolveStateDir } from "./state.mjs"; -export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; -export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; -export function createBrokerSessionDir(prefix = "cxc-") { - return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +function createBrokerSessionDir(prefix = "cxc-") { + const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + setMode(sessionDir, PRIVATE_DIR_MODE); + return sessionDir; } function connectToEndpoint(endpoint) { @@ -21,52 +42,148 @@ function connectToEndpoint(endpoint) { return net.createConnection({ path: target.path }); } -export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - const ready = await new Promise((resolve) => { - const socket = connectToEndpoint(endpoint); - socket.on("connect", () => { - socket.end(); - resolve(true); - }); - socket.on("error", () => resolve(false)); - }); - if (ready) { +/** + * One-shot connection probe. Resolves "connect" when something accepted the + * connection, "timeout" when nothing settled within timeoutMs, "invalid" when + * the endpoint cannot even be parsed, and the error code otherwise. + */ +function probeEndpoint(endpoint, timeoutMs) { + return new Promise((resolve) => { + let socket; + try { + socket = connectToEndpoint(endpoint); + } catch { + resolve("invalid"); + return; + } + let settled = false; + const finish = (value) => { + if (settled) { + return; + } + settled = true; + socket.destroy(); + resolve(value); + }; + // socket.setTimeout(0) disables the timer outright, which would leave this + // promise pending forever on a connection that never settles. + socket.setTimeout(Math.max(1, timeoutMs), () => finish("timeout")); + socket.on("connect", () => finish("connect")); + socket.on("error", (error) => finish(error?.code ?? "error")); + }); +} + +async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const probe = await probeEndpoint(endpoint, Math.min(150, deadline - Date.now())); + if (probe === "connect") { return true; } + if (probe === "invalid") { + return false; + } await new Promise((resolve) => setTimeout(resolve, 50)); } return false; } -export async function sendBrokerShutdown(endpoint) { - await new Promise((resolve) => { +export function sendBrokerShutdown(endpoint, options = {}) { + const timeoutMs = Number.isFinite(options.timeoutMs) + ? Math.max(1, options.timeoutMs) + : 2000; + return new Promise((resolve) => { const socket = connectToEndpoint(endpoint); + let settled = false; + let buffer = ""; + const finish = (result = null) => { + if (settled) { + return; + } + settled = true; + socket.destroy(); + resolve(result); + }; socket.setEncoding("utf8"); + socket.setTimeout(timeoutMs, () => finish(null)); socket.on("connect", () => { - socket.write(`${JSON.stringify({ id: 1, method: "broker/shutdown", params: {} })}\n`); + socket.write( + `${JSON.stringify({ + id: 1, + method: "broker/shutdown", + params: { instanceToken: options.instanceToken ?? null } + })}\n` + ); }); - socket.on("data", () => { - socket.end(); - resolve(); + socket.on("data", (chunk) => { + buffer += chunk; + const newlineIndex = buffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + try { + const response = JSON.parse(buffer.slice(0, newlineIndex)); + finish({ result: response?.result ?? null, error: response?.error ?? null }); + } catch { + finish(null); + } }); - socket.on("error", resolve); - socket.on("close", resolve); + socket.on("error", () => finish(null)); + socket.on("close", () => finish(null)); }); } -export function spawnBrokerProcess({ scriptPath, cwd, endpoint, pidFile, logFile, env = process.env }) { - const logFd = fs.openSync(logFile, "a"); - const child = spawn(process.execPath, [scriptPath, "serve", "--endpoint", endpoint, "--cwd", cwd, "--pid-file", pidFile], { - cwd, - env, - detached: true, - stdio: ["ignore", logFd, logFd] +function spawnBrokerProcess({ + scriptPath, + cwd, + endpoint, + pidFile, + logFile, + instanceToken, + env = process.env +}) { + const logFd = fs.openSync(logFile, "a", PRIVATE_FILE_MODE); + try { + setMode(logFile, PRIVATE_FILE_MODE); + const child = spawn( + process.execPath, + [ + scriptPath, + "serve", + "--endpoint", + endpoint, + "--cwd", + cwd, + "--pid-file", + pidFile, + "--instance-token", + instanceToken + ], + { + cwd, + env, + detached: true, + stdio: ["ignore", logFd, logFd] + } + ); + child.unref(); + return child; + } finally { + fs.closeSync(logFd); + } +} + +// child_process.spawn() can fail asynchronously (e.g. ENOENT when `cwd` does +// not exist): Node emits an "error" event on the next tick instead of +// throwing. Without a listener that event crashes the process, and callers +// that raced ahead to persist a tokenized session would leave a wedged +// broker.json (pid: null) behind. Callers must await this before treating the +// child as spawned. +function waitForBrokerSpawn(child) { + return new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); }); - child.unref(); - fs.closeSync(logFd); - return child; } function resolveBrokerStateFile(cwd) { @@ -74,13 +191,8 @@ function resolveBrokerStateFile(cwd) { } export function loadBrokerSession(cwd) { - const stateFile = resolveBrokerStateFile(cwd); - if (!fs.existsSync(stateFile)) { - return null; - } - try { - return JSON.parse(fs.readFileSync(stateFile, "utf8")); + return JSON.parse(fs.readFileSync(resolveBrokerStateFile(cwd), "utf8")); } catch { return null; } @@ -88,44 +200,363 @@ export function loadBrokerSession(cwd) { export function saveBrokerSession(cwd, session) { const stateDir = resolveStateDir(cwd); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(resolveBrokerStateFile(cwd), `${JSON.stringify(session, null, 2)}\n`, "utf8"); + ensurePrivateDir(stateDir); + writeJsonFileAtomic(resolveBrokerStateFile(cwd), session); +} + +function clearBrokerSession(cwd) { + removeFileIfExists(resolveBrokerStateFile(cwd)); } -export function clearBrokerSession(cwd) { - const stateFile = resolveBrokerStateFile(cwd); - if (fs.existsSync(stateFile)) { - fs.unlinkSync(stateFile); +function resolveBrokerPid(session) { + const statePid = isValidPid(session.pid) ? session.pid : null; + let filePid = null; + let rawPid = null; + if (session.pidFile) { + try { + rawPid = fs.readFileSync(session.pidFile, "utf8").trim(); + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + // A pid file that vanished is treated as absent; any other failure is + // an integrity problem and must abort instead of masking a mismatch. + } + } + if (rawPid !== null && /^\d+$/.test(rawPid)) { + const parsedPid = Number(rawPid); + filePid = isValidPid(parsedPid) ? parsedPid : null; + } + if (statePid && filePid && statePid !== filePid) { + throw new Error(`Codex app-server broker PID mismatch (${statePid} != ${filePid}).`); } + return statePid ?? filePid; } -async function isBrokerEndpointReady(endpoint) { +function endpointArtifactExists(endpoint) { if (!endpoint) { return false; } try { - return await waitForBrokerEndpoint(endpoint, 150); + const target = parseBrokerEndpoint(endpoint); + // Named pipes have no filesystem artifact for teardown to unlink. + return target.kind === "unix" && fs.existsSync(target.path); + } catch { + return true; + } +} + +function canDiscardUnownedSession(session, pid, options = {}) { + const processExited = !isValidPid(pid) || !isProcessTreeRunning(pid, options); + return processExited && !endpointArtifactExists(session.endpoint); +} + +/** + * Resolves false only when nothing can possibly be listening on the endpoint. + * + * A refused connection (or a socket path that is already gone) is the one signal + * that positively rules out a live listener. Every other outcome — including a + * timeout, which is what a live-but-hung broker produces — resolves true so that + * callers stay conservative and never unlink a socket that someone else owns. + */ +async function endpointAcceptsConnection(endpoint, timeoutMs) { + const deadlineMs = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 250; + const probe = await probeEndpoint(endpoint, deadlineMs); + return probe !== "ECONNREFUSED" && probe !== "ENOENT"; +} + +function resolveOwnedSessionDir(sessionDir) { + if (typeof sessionDir !== "string") { + return null; + } + const resolved = path.resolve(sessionDir); + const relative = path.relative(path.resolve(os.tmpdir()), resolved); + if (path.dirname(relative) !== "." || !path.basename(relative).startsWith("cxc-")) { + return null; + } + try { + const stats = fs.lstatSync(resolved); + return stats.isDirectory() && !stats.isSymbolicLink() ? resolved : null; + } catch { + return null; + } +} + +function endpointBelongsToSession(session, platform = process.platform) { + const sessionDir = resolveOwnedSessionDir(session.sessionDir); + if (!sessionDir || !session.endpoint) { + return false; + } + try { + return session.endpoint === createBrokerEndpoint(sessionDir, platform); } catch { return false; } } +function processMatchesLegacyBroker(session, pid, options = {}) { + const platform = options.platform ?? process.platform; + let expectedEndpoint; + try { + expectedEndpoint = createBrokerEndpoint(session.sessionDir, platform); + } catch { + return false; + } + if ( + !isValidPid(pid) || + session.endpoint !== expectedEndpoint || + typeof session.pidFile !== "string" || + session.pidFile !== path.join(session.sessionDir, "broker.pid") + ) { + return false; + } + const probeOptions = { + platform, + timeoutMs: options.timeoutMs, + runCommandImpl: options.runCommandImpl + }; + // The broker's original --cwd argument is not persisted, and the current + // invocation may address the same workspace through a different path, so + // ownership is proven by the launch artifacts unique to this session: its + // endpoint and its pid file inside the mkdtemp session directory. + return ( + processHasLaunchSequence(pid, ["serve", "--endpoint", session.endpoint], probeOptions) && + processHasLaunchSequence(pid, ["--pid-file", session.pidFile], probeOptions) + ); +} + +/** + * A stale socket left behind by an owned broker that died before acknowledging + * shutdown. Ownership cannot be proven by RPC in that case — the process that + * would answer is gone — so possession is established from independent signals + * instead, all of which must hold: + * + * 1. the socket sits inside the 0700 session directory we created; + * 2. the recorded leader is absent or replaced (see below for the + * process-group exception); + * 3. connecting to the endpoint is refused. + * + * No single one is sufficient. A hung broker also fails to answer, PID numbers + * get reused, and a refused connect only proves nothing is listening *right + * now* — a process that has bound but not yet listened also refuses. Requiring + * all three keeps the blast radius inside our own temp directory. + * + * `allowOrphanedTree` (default false) relaxes signal 2 from "process group + * gone" to "leader gone": the caller passes it only after already having + * established the abandoned-tokened-orphan state itself (a confirmed-dead or + * -replaced leader whose group still has members we deliberately never + * signal). It is safe here specifically because signal 3 still independently + * rules out any live listener on the endpoint, and a surviving group member + * never knew the socket path to begin with -- it was never handed the + * endpoint, so it cannot be the one refusing or accepting the connection. + * Legacy callers do not set the flag and keep the full process-group veto. + */ +async function canReclaimStaleEndpoint(session, pid, options = {}) { + if (!endpointBelongsToSession(session, options.platform)) { + return false; + } + if (isValidPid(pid) && isProcessRunning(pid, options)) { + return false; + } + // isProcessTreeRunning() checks the process *group* on Linux, so a reused PID + // in another group reads as dead. Pair it with the plain PID check above + // before treating the owner as gone -- unless the caller has already + // established the abandoned-orphan exception documented above. + if (!options.allowOrphanedTree && isValidPid(pid) && isProcessTreeRunning(pid, options)) { + return false; + } + return !(await endpointAcceptsConnection(session.endpoint, options.reclaimProbeTimeoutMs)); +} + +function processMatchesInstanceToken(pid, instanceToken, options) { + return processHasLaunchToken(pid, instanceToken, { ...options, marker: "--instance-token" }); +} + +// Legacy sessions have no instance token, so their processes are re-verified +// by launch artifacts; tokened sessions are re-verified by the token. +function ownsBrokerProcess(session, pid, legacySession, options) { + return legacySession + ? processMatchesLegacyBroker(session, pid, options) + : processMatchesInstanceToken(pid, session.instanceToken, options); +} + +export async function shutdownBrokerSession(cwd, options = {}) { + return withBrokerLock(cwd, options, () => shutdownBrokerSessionLocked(cwd, options)); +} + +async function shutdownBrokerSessionLocked(cwd, options = {}) { + const session = loadBrokerSession(cwd); + if (!session) { + return { found: false, exited: true, forced: false, reclaimedStaleEndpoint: false }; + } + + const pid = resolveBrokerPid(session); + // Liveness/exit checks against the persisted pid must be pinned to the + // process identity recorded at spawn time (when available), so a reused + // PID reads as exited instead of being mistaken for the original broker. + // Ownership checks (token or launch-artifact matching) are a separate, + // stronger proof and are deliberately left on the caller-supplied options. + // `!= null` alone would also accept "" or a non-string value that + // String()-coerces to something no live process could ever match, which + // would misread a live broker as replaced -- validate the shape first. + const recordedIdentity = + typeof session.processIdentity === "string" && session.processIdentity.length > 0 + ? session.processIdentity + : null; + const livenessOptions = recordedIdentity != null ? { ...options, identity: recordedIdentity } : options; + // Set when the persisted leader is confirmed gone (or replaced) while its + // process group still has survivors -- an abandoned, tokened orphan whose + // own state we may still reclaim without ever signaling the group. See the + // ownership-check branch below for the full rationale. + let abandonedOrphanedTree = false; + const legacySession = Boolean(session.endpoint && !session.instanceToken); + let legacyProcessVerified = false; + if (legacySession) { + if (canDiscardUnownedSession(session, pid, livenessOptions)) { + teardownAndClear(cwd, session, false); + return { found: true, exited: true, forced: false, reclaimedStaleEndpoint: false }; + } + legacyProcessVerified = processMatchesLegacyBroker(session, pid, options); + const legacyEndpointIsSafelyStale = + isValidPid(pid) && (await canReclaimStaleEndpoint(session, pid, livenessOptions)); + if (!legacyProcessVerified && !legacyEndpointIsSafelyStale) { + throw new Error("Codex app-server broker ownership could not be verified; persisted state was preserved."); + } + } + + let shutdownResponse = null; + if (session.endpoint) { + shutdownResponse = await sendBrokerShutdown(session.endpoint, { + timeoutMs: options.timeoutMs, + instanceToken: session.instanceToken + }); + } + if (shutdownResponse?.error) { + throw new Error( + `Codex app-server broker rejected shutdown identity; persisted state was preserved: ${ + shutdownResponse.error.message ?? "unknown error" + }` + ); + } + + const shutdownAck = shutdownResponse?.result ?? null; + const acknowledgedPid = isValidPid(shutdownAck?.pid) ? shutdownAck.pid : null; + const ownershipVerified = + (Boolean(session.instanceToken) && + shutdownAck?.instanceToken === session.instanceToken && + acknowledgedPid !== null && + (pid === null || pid === acknowledgedPid)) || + (legacySession && legacyProcessVerified && Boolean(shutdownAck)); + if (shutdownAck && !ownershipVerified) { + throw new Error("Codex app-server broker shutdown identity did not match persisted state."); + } + + let verifiedPid = ownershipVerified ? acknowledgedPid ?? pid : null; + let exited = isValidPid(pid) + ? await waitForProcessExit(pid, { ...livenessOptions, timeoutMs: 0 }) + : false; + + let processOwnershipProven = false; + if (!shutdownAck && isValidPid(pid) && !exited) { + const ownsPersistedProcess = ownsBrokerProcess(session, pid, legacySession, options); + if (!ownsPersistedProcess) { + if (isProcessTreeRunning(pid, livenessOptions)) { + if (!session.instanceToken || isProcessRunning(pid, livenessOptions)) { + throw new Error("Codex app-server broker ownership could not be verified; persisted state was preserved."); + } + // The leader is confirmed gone (or provably replaced) while its process + // group still has members. POSIX reserves a leader's pid for as long as + // the group survives, but a single observation cannot tell whether these + // members are the original broker's descendants or a later generation + // that recycled the same pgid after the original group died out -- so + // the group is deliberately never signaled. Reclaiming our own session + // state is still sound: nothing can be accepting on a refused endpoint, + // and the orphaned tree (which may still be finishing in-flight work, + // not necessarily idle) keeps running untouched. The alternative is + // wedging every later invocation in this workspace until the orphan + // exits on its own. + abandonedOrphanedTree = true; + } + exited = true; + } else { + verifiedPid = pid; + processOwnershipProven = true; + } + } + + if (!shutdownAck && session.instanceToken && !isValidPid(pid)) { + throw new Error("Codex app-server broker PID is unavailable; persisted ownership state was preserved."); + } + + if (!exited && isValidPid(verifiedPid)) { + exited = await waitForProcessExit(verifiedPid, livenessOptions); + } + + let forced = false; + if (!exited && isValidPid(verifiedPid) && options.killProcess) { + const stillOwnsProcess = ownsBrokerProcess(session, verifiedPid, legacySession, options); + if (!stillOwnsProcess) { + throw new Error("Codex app-server broker process ownership changed before forced shutdown."); + } + processOwnershipProven = true; + options.killProcess(verifiedPid); + forced = true; + exited = await waitForProcessExit(verifiedPid, livenessOptions); + } + + if (!exited) { + throw new Error(`Codex app-server broker ${verifiedPid ?? session.endpoint ?? "unknown"} did not exit.`); + } + // A broker that is killed after binding its socket can never send a shutdown + // ack, so ownershipVerified stays false while the socket file survives. Left + // fatal, that single stale socket wedges every later command in the workspace, + // because ensureBrokerSession() shuts the old session down before starting a + // replacement. Ownership proven against the live process (token or launch + // artifacts) already covers the endpoint; only an unproven leftover needs the + // connection probe, which can race the kernel right after a forced kill. + const endpointProven = ownershipVerified || processOwnershipProven; + let reclaimedStaleEndpoint = false; + if (!endpointProven && endpointArtifactExists(session.endpoint)) { + reclaimedStaleEndpoint = await canReclaimStaleEndpoint(session, pid, { + ...livenessOptions, + allowOrphanedTree: abandonedOrphanedTree + }); + if (!reclaimedStaleEndpoint) { + throw new Error("Codex app-server broker endpoint ownership could not be verified; persisted state was preserved."); + } + } + + const endpointIsOurs = endpointProven || reclaimedStaleEndpoint; + teardownAndClear(cwd, session, endpointIsOurs); + return { found: true, exited: true, forced, reclaimedStaleEndpoint }; +} + export async function ensureBrokerSession(cwd, options = {}) { + return withBrokerLock(cwd, options, () => ensureBrokerSessionLocked(cwd, options)); +} + +function withBrokerLock(cwd, options, action) { + const stateDir = resolveStateDir(cwd); + ensurePrivateDir(stateDir); + return withLock( + path.join(stateDir, ".broker.lock"), + action, + { timeoutMs: options.lockTimeoutMs ?? 10000 } + ); +} + +async function ensureBrokerSessionLocked(cwd, options = {}) { + const shutdownOptions = { + ...options, + killProcess: options.killProcess ?? terminateProcessTree + }; const existing = loadBrokerSession(cwd); - if (existing && (await isBrokerEndpointReady(existing.endpoint))) { + if (existing?.endpoint && (await waitForBrokerEndpoint(existing.endpoint, 150))) { return existing; } if (existing) { - teardownBrokerSession({ - endpoint: existing.endpoint ?? null, - pidFile: existing.pidFile ?? null, - logFile: existing.logFile ?? null, - sessionDir: existing.sessionDir ?? null, - pid: existing.pid ?? null, - killProcess: options.killProcess ?? null - }); - clearBrokerSession(cwd); + await shutdownBrokerSessionLocked(cwd, shutdownOptions); } const sessionDir = createBrokerSessionDir(); @@ -134,8 +565,8 @@ export async function ensureBrokerSession(cwd, options = {}) { const pidFile = path.join(sessionDir, "broker.pid"); const logFile = path.join(sessionDir, "broker.log"); const scriptPath = - options.scriptPath ?? - fileURLToPath(new URL("../app-server-broker.mjs", import.meta.url)); + options.scriptPath ?? fileURLToPath(new URL("../app-server-broker.mjs", import.meta.url)); + const instanceToken = options.instanceToken ?? randomUUID(); const child = spawnBrokerProcess({ scriptPath, @@ -143,20 +574,27 @@ export async function ensureBrokerSession(cwd, options = {}) { endpoint, pidFile, logFile, + instanceToken, env: options.env ?? process.env }); - const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); - if (!ready) { + // Persistence is deferred until the spawn is confirmed, so a session that + // never actually started never gets written with pid: null + a live + // instanceToken — that combination poisons shutdownBrokerSessionLocked() + // for every later invocation against this cwd (it throws "PID is + // unavailable" before any reclaim path runs), wedging the workspace. + try { + await waitForBrokerSpawn(child); + } catch (error) { teardownBrokerSession({ - endpoint, pidFile, logFile, sessionDir, - pid: child.pid ?? null, - killProcess: options.killProcess ?? null + ownershipVerified: false }); - return null; + throw new Error( + `Codex app-server broker failed to spawn: ${error?.message ?? "unknown error"}` + ); } const session = { @@ -164,46 +602,117 @@ export async function ensureBrokerSession(cwd, options = {}) { pidFile, logFile, sessionDir, - pid: child.pid ?? null + pid: child.pid ?? null, + // Cheap on Linux (one /proc read); a one-time ps/PowerShell call + // elsewhere. A null fallback degrades gracefully to today's behavior + // (no identity check) rather than failing session creation over it. + processIdentity: getProcessIdentity(child.pid) ?? null, + instanceToken }; - saveBrokerSession(cwd, session); - return session; -} - -export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { - if (Number.isFinite(pid) && killProcess) { - try { - killProcess(pid); - } catch { - // Ignore missing or already-exited broker processes. + try { + saveBrokerSession(cwd, session); + } catch (error) { + // The spawn already succeeded, so a failed persist would otherwise leave + // an untracked broker running with no state pointing at it. Killing and + // confirming exit before discarding artifacts avoids trading that leak + // for a worse one: deleting the socket/session dir out from under a + // child that is actually still alive because the kill was denied or slow. + // + // Build explicit options here instead of reusing shutdownOptions as-is: + // a caller-supplied identity must never leak into this check (we only + // ever want to compare against the identity we just recorded for this + // child), and shutdownOptions carries whatever the caller passed in. + const unwindOptions = { ...shutdownOptions, identity: session.processIdentity ?? undefined }; + // Only signal a pid that is provably still our child, mirroring the same + // ownership-before-kill invariant the forced-shutdown path already uses + // (see stillOwnsProcess above): the launch token in its argv is proof of + // possession, since an unrelated process that merely reused the pid + // cannot carry it. If the probe can't read the process at all (already + // gone, or unreadable), the gate simply skips the kill and the + // wait/preserve branch below handles it loudly either way. + if (processMatchesInstanceToken(child.pid, instanceToken, options)) { + try { + shutdownOptions.killProcess(child.pid); + } catch { + // A failure to signal the child is secondary to the persist error; + // waitForProcessExit below still tells us whether it actually died. + } + } + const childExited = await waitForProcessExit(child.pid, unwindOptions); + if (!childExited) { + throw new Error( + `Codex app-server broker session could not be persisted (${ + error?.message ?? "unknown error" + }), and the spawned process ${child.pid} did not exit; session artifacts were preserved.` + ); } + teardownBrokerSession({ + endpoint, + pidFile, + logFile, + sessionDir, + ownershipVerified: true + }); + throw error; } - if (pidFile && fs.existsSync(pidFile)) { - fs.unlinkSync(pidFile); + const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); + if (!ready) { + await shutdownBrokerSessionLocked(cwd, shutdownOptions); + return null; } - if (logFile && fs.existsSync(logFile)) { - fs.unlinkSync(logFile); + return session; +} + +function teardownAndClear(cwd, session, endpointIsOurs) { + teardownBrokerSession({ + endpoint: endpointIsOurs ? session.endpoint ?? null : null, + pidFile: session.pidFile ?? null, + logFile: session.logFile ?? null, + sessionDir: session.sessionDir ?? null, + ownershipVerified: endpointIsOurs + }); + clearBrokerSession(cwd); +} + +function teardownBrokerSession({ + endpoint = null, + pidFile, + logFile, + sessionDir = null, + ownershipVerified = false +}) { + if (endpoint && !ownershipVerified) { + throw new Error("Refusing to remove an unverified broker endpoint."); } - if (endpoint) { - try { - const target = parseBrokerEndpoint(endpoint); - if (target.kind === "unix" && fs.existsSync(target.path)) { - fs.unlinkSync(target.path); - } - } catch { - // Ignore malformed or already-removed broker endpoints during teardown. + const ownedSessionDir = resolveOwnedSessionDir(sessionDir); + const files = []; + if (ownedSessionDir && pidFile === path.join(ownedSessionDir, "broker.pid")) { + files.push(pidFile); + } + if (ownedSessionDir && logFile === path.join(ownedSessionDir, "broker.log")) { + files.push(logFile); + } + if (ownedSessionDir && endpoint === createBrokerEndpoint(ownedSessionDir)) { + const target = parseBrokerEndpoint(endpoint); + if (target.kind === "unix") { + files.push(target.path); } } - const resolvedSessionDir = sessionDir ?? (pidFile ? path.dirname(pidFile) : logFile ? path.dirname(logFile) : null); - if (resolvedSessionDir && fs.existsSync(resolvedSessionDir)) { + for (const filePath of files) { + removeFileIfExists(filePath); + } + + if (ownedSessionDir) { try { - fs.rmdirSync(resolvedSessionDir); - } catch { - // Ignore non-empty or missing directories. + fs.rmdirSync(ownedSessionDir); + } catch (error) { + if (error?.code !== "ENOENT" && error?.code !== "ENOTEMPTY") { + throw error; + } } } } diff --git a/plugins/codex/scripts/lib/fs.mjs b/plugins/codex/scripts/lib/fs.mjs index 027522442..730259831 100644 --- a/plugins/codex/scripts/lib/fs.mjs +++ b/plugins/codex/scripts/lib/fs.mjs @@ -1,6 +1,24 @@ +import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import process from "node:process"; + +export const PRIVATE_DIR_MODE = 0o700; +export const PRIVATE_FILE_MODE = 0o600; + +export function setMode(filePath, mode) { + try { + fs.chmodSync(filePath, mode); + } catch { + // Windows and restrictive filesystems may not implement POSIX modes. + } +} + +export function ensurePrivateDir(dir) { + fs.mkdirSync(dir, { recursive: true, mode: PRIVATE_DIR_MODE }); + setMode(dir, PRIVATE_DIR_MODE); +} export function ensureAbsolutePath(cwd, maybePath) { return path.isAbsolute(maybePath) ? maybePath : path.resolve(cwd, maybePath); @@ -14,8 +32,45 @@ export function readJsonFile(filePath) { return JSON.parse(fs.readFileSync(filePath, "utf8")); } -export function writeJsonFile(filePath, value) { - fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +export function writePrivateFile(filePath, value) { + fs.writeFileSync(filePath, value, { encoding: "utf8", mode: PRIVATE_FILE_MODE }); + setMode(filePath, PRIVATE_FILE_MODE); +} + +export function writeJsonFileAtomic(filePath, value) { + const temporaryFile = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + try { + const fd = fs.openSync(temporaryFile, "wx", PRIVATE_FILE_MODE); + try { + try { + fs.fchmodSync(fd, PRIVATE_FILE_MODE); + } catch { + // Windows and restrictive filesystems may not implement POSIX modes. + } + fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, "utf8"); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(temporaryFile, filePath); + setMode(filePath, PRIVATE_FILE_MODE); + } catch (error) { + fs.rmSync(temporaryFile, { force: true }); + throw error; + } +} + +export function removeFileIfExists(filePath) { + if (!filePath) { + return; + } + try { + fs.unlinkSync(filePath); + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + } } export function safeReadFile(filePath) { diff --git a/plugins/codex/scripts/lib/locking.mjs b/plugins/codex/scripts/lib/locking.mjs new file mode 100644 index 000000000..9074aac62 --- /dev/null +++ b/plugins/codex/scripts/lib/locking.mjs @@ -0,0 +1,263 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +import { getProcessIdentity, isProcessRunning } from "./process.mjs"; + +const DEFAULT_TIMEOUT_MS = 5000; +const DEFAULT_STALE_MS = 30000; +const RETRY_DELAY_MS = 25; +const OWNER_FILE_PREFIX = "owner-"; +const OWNER_FILE_SUFFIX = ".json"; +// Start-time lookup may spawn `ps` or PowerShell off Linux. Compute it lazily, +// on first lock acquisition, and cache it for the rest of the process instead +// of paying that cost on import for every short state update. +let processIdentity; +function cachedProcessIdentity() { + if (processIdentity === undefined) { + processIdentity = getProcessIdentity(process.pid); + } + return processIdentity; +} + +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function ownerFileName(token) { + return `${OWNER_FILE_PREFIX}${token}${OWNER_FILE_SUFFIX}`; +} + +function writePrivateJson(filePath, value) { + const fd = fs.openSync(filePath, "wx", 0o600); + try { + fs.writeFileSync(fd, `${JSON.stringify(value)}\n`, "utf8"); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function removeCandidate(handle) { + try { + fs.unlinkSync(handle.candidateOwnerFile); + } catch { + // Candidate may already have been published or cleaned up. + } + try { + fs.rmdirSync(handle.candidateDir); + } catch { + // Candidate may already have been published or cleaned up. + } +} + +function tryAcquire(lockDir) { + const token = randomUUID(); + const candidateDir = `${lockDir}.candidate-${token}`; + const candidateOwnerFile = path.join(candidateDir, ownerFileName(token)); + const handle = { + lockDir, + token, + ownerFile: path.join(lockDir, ownerFileName(token)), + candidateDir, + candidateOwnerFile + }; + + // Resolve identity before creating the candidate so a crash during a slow + // ps/PowerShell lookup cannot leave an orphaned candidate directory behind. + const identity = cachedProcessIdentity(); + fs.mkdirSync(candidateDir, { mode: 0o700 }); + try { + writePrivateJson(candidateOwnerFile, { + pid: process.pid, + token, + createdAt: Date.now(), + processIdentity: identity + }); + fs.renameSync(candidateDir, lockDir); + return handle; + } catch (error) { + removeCandidate(handle); + const targetExists = fs.existsSync(lockDir); + if ( + error?.code === "EEXIST" || + error?.code === "ENOTEMPTY" || + (error?.code === "EPERM" && targetExists) + ) { + return null; + } + throw error; + } +} + +function readLockState(lockDir) { + let entries; + let ageMs; + try { + entries = fs.readdirSync(lockDir); + ageMs = Date.now() - fs.statSync(lockDir).mtimeMs; + } catch { + return { kind: "missing" }; + } + + const ownerFiles = entries.filter( + (entry) => entry.startsWith(OWNER_FILE_PREFIX) && entry.endsWith(OWNER_FILE_SUFFIX) + ); + if (ownerFiles.length === 0 && entries.length === 0) { + return { kind: "empty", ageMs }; + } + if (ownerFiles.length !== 1 || entries.length !== 1) { + return { kind: "corrupt", ageMs }; + } + + const fileName = ownerFiles[0]; + const ownerFile = path.join(lockDir, fileName); + try { + const owner = JSON.parse(fs.readFileSync(ownerFile, "utf8")); + const expectedFileName = ownerFileName(owner.token); + if ( + !Number.isSafeInteger(owner.pid) || + owner.pid <= 0 || + typeof owner.token !== "string" || + owner.token.length === 0 || + (owner.processIdentity != null && typeof owner.processIdentity !== "string") || + fileName !== expectedFileName + ) { + return { kind: "corrupt", ageMs }; + } + return { kind: "owned", owner, ownerFile, ageMs }; + } catch { + return { kind: "corrupt", ageMs }; + } +} + +function removeOwnedLock(ownerFile, lockDir) { + try { + // The token is part of the filename, so an old owner/reclaimer cannot + // unlink a successor's ownership record. + fs.unlinkSync(ownerFile); + } catch { + return false; + } + try { + fs.rmdirSync(lockDir); + } catch { + // A successor may already have atomically replaced the now-empty dir. + } + return true; +} + +function reclaimAbandonedLock(lockDir, options) { + const state = readLockState(lockDir); + if (state.kind === "missing") { + return true; + } + if (state.kind === "owned") { + const processRunning = options.isProcessRunning ?? isProcessRunning; + const processIdentity = state.owner.processIdentity ?? null; + if ( + processRunning(state.owner.pid, { + identity: processIdentity ?? undefined + }) + ) { + // When the identity lookup failed at acquire time the owner records + // null. Once such a short-lived critical section looks stale, a live + // PID alone cannot distinguish the original owner from an unrelated + // process that reused the PID, so reclaiming is the lesser risk: + // refusing would wedge every later caller behind a reused PID. + if (processIdentity == null && state.ageMs > options.staleMs) { + return removeOwnedLock(state.ownerFile, lockDir); + } + return false; + } + return removeOwnedLock(state.ownerFile, lockDir); + } + if (state.kind === "empty" && state.ageMs > options.staleMs) { + try { + fs.rmdirSync(lockDir); + return true; + } catch { + return false; + } + } + // Unknown non-empty contents fail closed. Published locks always contain + // one complete owner file, so silently deleting anything else is unsafe. + return false; +} + +function normalizeOptions(options) { + return { + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + staleMs: options.staleMs ?? DEFAULT_STALE_MS, + retryDelayMs: options.retryDelayMs ?? RETRY_DELAY_MS, + isProcessRunning: options.isProcessRunning + }; +} + +// Returns a handle when the lock was acquired, null when the caller should +// sleep and retry, and throws once the deadline has passed. +function tryAcquireOnce(lockDir, normalized, deadline) { + const handle = tryAcquire(lockDir); + if (handle) { + return handle; + } + reclaimAbandonedLock(lockDir, normalized); + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for lock: ${lockDir}`); + } + return null; +} + +export function acquireLockSync(lockDir, options = {}) { + const normalized = normalizeOptions(options); + const deadline = Date.now() + normalized.timeoutMs; + while (true) { + const handle = tryAcquireOnce(lockDir, normalized, deadline); + if (handle) { + return handle; + } + sleepSync(normalized.retryDelayMs); + } +} + +export async function acquireLock(lockDir, options = {}) { + const normalized = normalizeOptions(options); + const deadline = Date.now() + normalized.timeoutMs; + while (true) { + const handle = tryAcquireOnce(lockDir, normalized, deadline); + if (handle) { + return handle; + } + await sleep(normalized.retryDelayMs); + } +} + +export function releaseLock(handle) { + if (!handle?.lockDir || !handle?.ownerFile) { + return false; + } + return removeOwnedLock(handle.ownerFile, handle.lockDir); +} + +export function withLockSync(lockDir, fn, options = {}) { + const handle = acquireLockSync(lockDir, options); + try { + return fn(); + } finally { + releaseLock(handle); + } +} + +export async function withLock(lockDir, fn, options = {}) { + const handle = await acquireLock(lockDir, options); + try { + return await fn(); + } finally { + releaseLock(handle); + } +} diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index dd8fc3751..c7396915b 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import fs from "node:fs"; import process from "node:process"; export function runCommand(command, args = [], options = {}) { @@ -8,6 +9,8 @@ export function runCommand(command, args = [], options = {}) { encoding: "utf8", input: options.input, maxBuffer: options.maxBuffer, + timeout: options.timeout, + killSignal: options.killSignal, stdio: options.stdio ?? "pipe", shell: options.shell ?? (process.platform === "win32" ? (process.env.SHELL || true) : false), windowsHide: true @@ -16,7 +19,9 @@ export function runCommand(command, args = [], options = {}) { return { command, args, - status: result.status ?? 0, + // Preserve Node's spawnSync contract: signal-terminated commands have a + // null status and a non-null signal. + status: result.status, signal: result.signal ?? null, stdout: result.stdout ?? "", stderr: result.stderr ?? "", @@ -29,7 +34,7 @@ export function runCommandChecked(command, args = [], options = {}) { if (result.error) { throw result.error; } - if (result.status !== 0) { + if (result.signal != null || result.status !== 0) { throw new Error(formatCommandFailure(result)); } return result; @@ -43,8 +48,11 @@ export function binaryAvailable(command, versionArgs = ["--version"], options = if (result.error) { return { available: false, detail: result.error.message }; } - if (result.status !== 0) { - const detail = result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`; + if (result.signal != null || result.status !== 0) { + const detail = + result.stderr.trim() || + result.stdout.trim() || + (result.signal != null ? `signal ${result.signal}` : `exit ${result.status}`); return { available: false, detail }; } return { available: true, detail: result.stdout.trim() || result.stderr.trim() || "ok" }; @@ -54,8 +62,319 @@ function looksLikeMissingProcessMessage(text) { return /not found|no running instance|cannot find|does not exist|no such process/i.test(text); } +export function isValidPid(pid) { + return Number.isSafeInteger(pid) && pid > 0; +} + +function readLinuxProcessStat(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd === -1) { + return null; + } + const fields = stat.slice(commandEnd + 1).trim().split(/\s+/); + return { + state: fields[0] ?? null, + processGroup: fields[2] ?? null, + // /proc//stat field 22; fields starts at field 3. + startTime: fields[19] ?? null + }; + } catch { + return null; + } +} + +function queryProcessTable(pid, powershellCommand, psFormat, options) { + const runCommandImpl = options.runCommandImpl ?? runCommand; + const spawnOptions = { + timeout: options.timeoutMs ?? 2000, + killSignal: "SIGTERM", + // ps -o lstart= renders via the C library's %c, which is locale- and + // TZ-dependent: the same instant can print differently between the + // recording and the checking invocation, producing a false mismatch on a + // still-live broker. Force a fixed, unambiguous rendering. Harmless for + // the PowerShell path (already UTC ticks) and for command-line reads. + env: { ...(options.env ?? process.env), LC_ALL: "C", TZ: "UTC" } + }; + const result = + (options.platform ?? process.platform) === "win32" + ? runCommandImpl( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-Command", powershellCommand], + spawnOptions + ) + : runCommandImpl("ps", ["-ww", "-p", String(pid), "-o", psFormat], spawnOptions); + if (result.error || result.signal != null || result.status !== 0) { + return null; + } + return String(result.stdout ?? ""); +} + +// /proc//stat's starttime is in clock ticks since BOOT, not since the +// epoch, so the bare value collides across reboots (a reused pid after a +// reboot can present the same tick count as an earlier, unrelated process). +// Prefixing with the boot id disambiguates across reboots while keeping +// identities comparable within one. Read once and cache for the process +// lifetime -- it cannot change without a reboot, which also invalidates any +// identity recorded before it. Only a SUCCESSFUL read is cached: a transient +// failure (e.g. a sandboxed /proc) must not poison every later call for the +// rest of the process's lifetime -- each subsequent call gets another chance +// to read it. +let cachedBootId; +function bootId() { + if (cachedBootId === undefined) { + try { + cachedBootId = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + } catch { + return ""; + } + } + return cachedBootId; +} + +export function getProcessIdentity(pid, options = {}) { + if (!isValidPid(pid)) { + return null; + } + if ((options.platform ?? process.platform) === "linux") { + const startTime = readLinuxProcessStat(pid)?.startTime ?? null; + if (startTime == null) { + return null; + } + const boot = bootId(); + if (!boot) { + // boot_id unreadable: a raw tick count is boot-relative and would both + // collide across boots and falsely mismatch against composite values + // recorded when boot_id WAS readable. Unknown must never masquerade + // as a comparable identity. + return null; + } + return `${boot}:${startTime}`; + } + const output = queryProcessTable( + pid, + `$p = Get-Process -Id ${pid} -ErrorAction SilentlyContinue; if ($null -ne $p) { [Console]::Out.Write($p.StartTime.ToUniversalTime().Ticks) }`, + "lstart=", + options + ); + return output?.trim() || null; +} + +export function isProcessRunning(pid, options = {}) { + if (!isValidPid(pid)) { + return false; + } + + const platform = options.platform ?? process.platform; + const killImpl = options.killImpl ?? process.kill.bind(process); + try { + killImpl(pid, 0); + } catch (error) { + if (error?.code === "EPERM") { + return true; + } + if (error?.code === "ESRCH") { + return false; + } + throw error; + } + + if (platform === "linux") { + const readProcessStat = options.readProcessStat ?? readLinuxProcessStat; + const stat = readProcessStat(pid); + // A null stat means /proc could not be inspected even though kill(pid, 0) + // proved the process exists. Failure to inspect is not proof of exit or + // replacement, so stay conservative and keep reporting it as running. + if (stat) { + // Zombies still answer kill(pid, 0), but no longer own a live resource. + if (stat.state === "Z" || stat.state === "X") { + return false; + } + if (options.identity != null) { + // getProcessIdentity() prefixes Linux identities with the boot id + // (see its comment); build the same composite here so this direct + // stat.startTime comparison stays consistent with values recorded + // through getProcessIdentity(). Only compare when boot_id is + // currently readable and the composite can actually be formed -- + // a bare tick count is boot-relative and must never be compared + // directly, which would both collide across boots and falsely + // mismatch a still-live process against a composite identity + // recorded when boot_id WAS readable. + const boot = bootId(); + if (boot) { + const currentIdentity = `${boot}:${stat.startTime}`; + if (currentIdentity !== String(options.identity)) { + return false; + } + } + // boot_id unreadable right now: the comparison cannot be formed, so + // it proves nothing. Failure to inspect is not proof of replacement. + } + } + } else if (options.identity != null) { + const currentIdentity = getProcessIdentity(pid, options); + // Failure to inspect a live process is not proof that it was replaced. + if (currentIdentity != null && currentIdentity !== String(options.identity)) { + return false; + } + } + + return true; +} + +function isLinuxProcessGroupRunning(pid) { + let entries; + try { + entries = fs.readdirSync("/proc", { withFileTypes: true }); + } catch { + return isProcessRunning(pid); + } + for (const entry of entries) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) { + continue; + } + const stat = readLinuxProcessStat(Number(entry.name)); + if ( + stat?.processGroup === String(pid) && + stat.state !== "Z" && + stat.state !== "X" + ) { + return true; + } + } + return false; +} + +export function isProcessTreeRunning(pid, options = {}) { + if (!isValidPid(pid)) { + return false; + } + const platform = options.platform ?? process.platform; + if (platform === "win32") { + return isProcessRunning(pid, options); + } + if (options.identity != null) { + const currentIdentity = getProcessIdentity(pid, options); + if (currentIdentity != null && currentIdentity !== String(options.identity)) { + // The pid exists and belongs to a different process. POSIX keeps a group + // leader's pid reserved while its group still has members, so a confirmed + // replacement also proves the recorded group is gone. + return false; + } + // An absent or unreadable pid proves nothing about survivors in its group; + // fall through to the group checks below. In particular, a leader that + // has already exited (identity now unreadable) must not short-circuit + // here: its process group can still have live descendants. + } + if (platform === "linux" && !options.killImpl) { + return isLinuxProcessGroupRunning(pid); + } + + const killImpl = options.killImpl ?? process.kill.bind(process); + try { + killImpl(-pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") { + return false; + } + if (error?.code === "EPERM") { + return true; + } + throw error; + } +} + +export async function waitForProcessExit(pid, options = {}) { + if (!isValidPid(pid)) { + return false; + } + const timeoutMs = options.timeoutMs ?? 2000; + const intervalMs = options.intervalMs ?? 25; + const isRunning = options.tree === false ? isProcessRunning : isProcessTreeRunning; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isRunning(pid, options)) { + return true; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + return !isRunning(pid, options); +} + +function readProcessCommandLine(pid, options) { + return queryProcessTable( + pid, + `$p = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}'; if ($null -ne $p) { [Console]::Out.Write($p.CommandLine) }`, + "command=", + options + ); +} + +function readLinuxCommandLineArgs(pid) { + try { + return fs.readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0"); + } catch { + return null; + } +} + +export function processHasLaunchSequence(pid, expectedArgs, options = {}) { + if ( + !isValidPid(pid) || + !Array.isArray(expectedArgs) || + expectedArgs.length === 0 || + expectedArgs.some((arg) => typeof arg !== "string" || arg.length === 0) + ) { + return false; + } + + if ((options.platform ?? process.platform) === "linux") { + const argv = readLinuxCommandLineArgs(pid)?.filter(Boolean); + if (!argv) { + return false; + } + return argv.some((_, start) => + expectedArgs.every((expected, offset) => argv[start + offset] === expected) + ); + } + + const commandLine = readProcessCommandLine(pid, options); + if (commandLine == null) { + return false; + } + let cursor = 0; + for (const expected of expectedArgs) { + const index = commandLine.indexOf(expected, cursor); + if (index === -1) { + return false; + } + cursor = index + expected.length; + } + return true; +} + +export function processHasLaunchToken(pid, token, options = {}) { + if (!isValidPid(pid) || typeof token !== "string" || token.length < 16) { + return false; + } + + const marker = options.marker ?? "--worker-token"; + if ((options.platform ?? process.platform) === "linux") { + const argv = readLinuxCommandLineArgs(pid); + return argv != null && argv.includes(marker) && argv.includes(token); + } + + const commandLine = readProcessCommandLine(pid, options); + if (commandLine == null) { + return false; + } + return commandLine.includes(marker) && commandLine.includes(token); +} + export function terminateProcessTree(pid, options = {}) { - if (!Number.isFinite(pid)) { + if (!isValidPid(pid)) { return { attempted: false, delivered: false, method: null }; } diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 2da23498f..f6fdf5e2a 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -3,6 +3,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { + ensurePrivateDir, + removeFileIfExists, + writeJsonFileAtomic, + writePrivateFile +} from "./fs.mjs"; +import { withLockSync } from "./locking.mjs"; import { resolveWorkspaceRoot } from "./workspace.mjs"; const STATE_VERSION = 1; @@ -52,7 +59,11 @@ export function resolveJobsDir(cwd) { } export function ensureStateDir(cwd) { - fs.mkdirSync(resolveJobsDir(cwd), { recursive: true }); + const stateDir = resolveStateDir(cwd); + const jobsDir = resolveJobsDir(cwd); + for (const dir of [stateDir, jobsDir]) { + ensurePrivateDir(dir); + } } export function loadState(cwd) { @@ -83,15 +94,12 @@ function pruneJobs(jobs) { .slice(0, MAX_JOBS); } -function removeFileIfExists(filePath) { - if (filePath && fs.existsSync(filePath)) { - fs.unlinkSync(filePath); - } +function resolveStateLockDir(cwd) { + return path.join(resolveStateDir(cwd), ".state.lock"); } -export function saveState(cwd, state) { +function saveStateLocked(cwd, state) { const previousJobs = loadState(cwd).jobs; - ensureStateDir(cwd); const nextJobs = pruneJobs(state.jobs ?? []); const nextState = { version: STATE_VERSION, @@ -107,18 +115,26 @@ export function saveState(cwd, state) { if (retainedIds.has(job.id)) { continue; } - removeJobFile(resolveJobFile(cwd, job.id)); + removeFileIfExists(resolveJobFile(cwd, job.id)); removeFileIfExists(job.logFile); } - fs.writeFileSync(resolveStateFile(cwd), `${JSON.stringify(nextState, null, 2)}\n`, "utf8"); + writeJsonFileAtomic(resolveStateFile(cwd), nextState); return nextState; } +export function saveState(cwd, state) { + ensureStateDir(cwd); + return withLockSync(resolveStateLockDir(cwd), () => saveStateLocked(cwd, state)); +} + export function updateState(cwd, mutate) { - const state = loadState(cwd); - mutate(state); - return saveState(cwd, state); + ensureStateDir(cwd); + return withLockSync(resolveStateLockDir(cwd), () => { + const state = loadState(cwd); + mutate(state); + return saveStateLocked(cwd, state); + }); } export function generateJobId(prefix = "job") { @@ -166,7 +182,7 @@ export function getConfig(cwd) { export function writeJobFile(cwd, jobId, payload) { ensureStateDir(cwd); const jobFile = resolveJobFile(cwd, jobId); - fs.writeFileSync(jobFile, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + writePrivateFile(jobFile, `${JSON.stringify(payload, null, 2)}\n`); return jobFile; } @@ -174,12 +190,6 @@ export function readJobFile(jobFile) { return JSON.parse(fs.readFileSync(jobFile, "utf8")); } -function removeJobFile(jobFile) { - if (fs.existsSync(jobFile)) { - fs.unlinkSync(jobFile); - } -} - export function resolveJobLogFile(cwd, jobId) { ensureStateDir(cwd); return path.join(resolveJobsDir(cwd), `${jobId}.log`); diff --git a/plugins/codex/scripts/lib/tracked-jobs.mjs b/plugins/codex/scripts/lib/tracked-jobs.mjs index 902869012..e6b913aaa 100644 --- a/plugins/codex/scripts/lib/tracked-jobs.mjs +++ b/plugins/codex/scripts/lib/tracked-jobs.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import process from "node:process"; +import { writePrivateFile } from "./fs.mjs"; import { readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; @@ -50,7 +51,7 @@ export function appendLogBlock(logFile, title, body) { export function createJobLogFile(workspaceRoot, jobId, title) { const logFile = resolveJobLogFile(workspaceRoot, jobId); - fs.writeFileSync(logFile, "", "utf8"); + writePrivateFile(logFile, ""); if (title) { appendLogLine(logFile, `Starting ${title}.`); } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 778571e6c..fa4ee67fa 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -4,16 +4,8 @@ import fs from "node:fs"; import process from "node:process"; import { terminateProcessTree } from "./lib/process.mjs"; -import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; -import { - clearBrokerSession, - LOG_FILE_ENV, - loadBrokerSession, - PID_FILE_ENV, - sendBrokerShutdown, - teardownBrokerSession -} from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; +import { shutdownBrokerSession } from "./lib/broker-lifecycle.mjs"; +import { resolveStateFile, updateState } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; @@ -50,27 +42,21 @@ function cleanupSessionJobs(cwd, sessionId) { return; } - const state = loadState(workspaceRoot); - const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); - if (removedJobs.length === 0) { - return; - } - - for (const job of removedJobs) { - const stillRunning = job.status === "queued" || job.status === "running"; - if (!stillRunning) { - continue; + updateState(workspaceRoot, (state) => { + const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); + for (const job of removedJobs) { + const stillRunning = job.status === "queued" || job.status === "running"; + if (!stillRunning) { + continue; + } + try { + terminateProcessTree(job.pid ?? Number.NaN); + } catch { + // Ignore teardown failures during session shutdown. + } } - try { - terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. - } - } - saveState(workspaceRoot, { - ...state, - jobs: state.jobs.filter((job) => job.sessionId !== sessionId) + state.jobs = state.jobs.filter((job) => job.sessionId !== sessionId); }); } @@ -82,35 +68,25 @@ function handleSessionStart(input) { async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); - const brokerSession = - loadBrokerSession(cwd) ?? - (process.env[BROKER_ENDPOINT_ENV] - ? { - endpoint: process.env[BROKER_ENDPOINT_ENV], - pidFile: process.env[PID_FILE_ENV] ?? null, - logFile: process.env[LOG_FILE_ENV] ?? null - } - : null); - const brokerEndpoint = brokerSession?.endpoint ?? null; - const pidFile = brokerSession?.pidFile ?? null; - const logFile = brokerSession?.logFile ?? null; - const sessionDir = brokerSession?.sessionDir ?? null; - const pid = brokerSession?.pid ?? null; - - if (brokerEndpoint) { - await sendBrokerShutdown(brokerEndpoint); + const failures = []; + + try { + cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); + } catch (error) { + failures.push(error); } - cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - teardownBrokerSession({ - endpoint: brokerEndpoint, - pidFile, - logFile, - sessionDir, - pid, - killProcess: terminateProcessTree - }); - clearBrokerSession(cwd); + try { + await shutdownBrokerSession(cwd, { + killProcess: terminateProcessTree + }); + } catch (error) { + failures.push(error); + } + + if (failures.length > 0) { + throw new AggregateError(failures, `Codex session cleanup did not finish: ${failures.join("; ")}`); + } } async function main() { diff --git a/tests/broker-lifecycle.test.mjs b/tests/broker-lifecycle.test.mjs new file mode 100644 index 000000000..f2a047e55 --- /dev/null +++ b/tests/broker-lifecycle.test.mjs @@ -0,0 +1,1090 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import test from "node:test"; +import { spawn } from "node:child_process"; + +import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; +import { makeTempDir } from "./helpers.mjs"; +import { createBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; +import { + ensureBrokerSession, + loadBrokerSession, + saveBrokerSession, + sendBrokerShutdown, + shutdownBrokerSession +} from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { + getProcessIdentity, + isProcessRunning, + isProcessTreeRunning, + terminateProcessTree, + waitForProcessExit +} from "../plugins/codex/scripts/lib/process.mjs"; +import { + acquireLock, + releaseLock +} from "../plugins/codex/scripts/lib/locking.mjs"; +import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; + +test("concurrent ensureBrokerSession calls share a single authenticated broker", async () => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir); + + const [left, right] = await Promise.all([ + ensureBrokerSession(workspace, { env }), + ensureBrokerSession(workspace, { env }) + ]); + + assert.ok(left, "first ensureBrokerSession returned no session"); + assert.ok(right, "second ensureBrokerSession returned no session"); + assert.equal(left.endpoint, right.endpoint); + assert.equal(left.pid, right.pid); + assert.equal(left.instanceToken, right.instanceToken); + + const persisted = loadBrokerSession(workspace); + assert.ok(persisted); + assert.equal(persisted.endpoint, left.endpoint); + assert.equal(persisted.instanceToken, left.instanceToken); + + const outcome = await shutdownBrokerSession(workspace, { + killProcess: terminateProcessTree + }); + assert.equal(outcome.exited, true); + assert.equal(loadBrokerSession(workspace), null); +}); + +test("shutdown waits for the broker lock and reloads persisted state", async () => { + const workspace = makeTempDir(); + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(stateDir, { recursive: true }); + const lock = await acquireLock(path.join(stateDir, ".broker.lock")); + let settled = false; + + try { + const shutdown = shutdownBrokerSession(workspace); + shutdown.then( + () => { + settled = true; + }, + () => { + settled = true; + } + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(settled, false, "shutdown bypassed the broker lifecycle lock"); + + saveBrokerSession(workspace, { + endpoint: `unix:${path.join(workspace, "missing.sock")}`, + pid: null, + pidFile: null, + logFile: null, + sessionDir: null + }); + releaseLock(lock); + + const outcome = await shutdown; + assert.equal(outcome.found, true); + assert.equal(loadBrokerSession(workspace), null); + } finally { + releaseLock(lock); + } +}); + +test("broker rejects a shutdown token that does not identify its instance", async (t) => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const session = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + assert.ok(session?.pid); + + t.after(async () => { + if (loadBrokerSession(workspace)) { + await shutdownBrokerSession(workspace, { + killProcess: terminateProcessTree + }); + } + }); + + const response = await sendBrokerShutdown(session.endpoint, { + instanceToken: "wrong-instance-token", + timeoutMs: 500 + }); + + assert.match(response?.error?.message ?? "", /identity did not match/i); + assert.equal(isProcessTreeRunning(session.pid), true); + assert.ok(loadBrokerSession(workspace)); +}); + +test("shutdown closes idle half-open broker clients", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const session = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + const idleClient = net.createConnection({ + path: session.endpoint.slice("unix:".length), + allowHalfOpen: true + }); + idleClient.on("end", () => {}); + await new Promise((resolve, reject) => { + idleClient.once("connect", resolve); + idleClient.once("error", reject); + }); + t.after(() => idleClient.destroy()); + + const outcome = await shutdownBrokerSession(workspace, { + timeoutMs: 500, + intervalMs: 10, + killProcess: terminateProcessTree + }); + + assert.equal(outcome.exited, true); + assert.equal(outcome.forced, false); + assert.equal(loadBrokerSession(workspace), null); +}); + +test("shutdown request always uses a finite deadline", { skip: process.platform === "win32" }, async (t) => { + const sessionDir = makeTempDir("cxc-unresponsive-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("data", () => { + // Deliberately never answer. + }); + }); + await new Promise((resolve) => server.listen(socketPath, resolve)); + t.after(async () => { + for (const socket of sockets) { + socket.destroy(); + } + await new Promise((resolve) => server.close(resolve)); + }); + + for (const timeoutMs of [0, 40]) { + const startedAt = Date.now(); + const response = await sendBrokerShutdown(`unix:${socketPath}`, { + instanceToken: "instance-token-1234567890", + timeoutMs + }); + assert.equal(response, null); + assert.ok(Date.now() - startedAt < 500, "shutdown request exceeded its deadline"); + } + assert.equal(fs.existsSync(socketPath), true); +}); + +test("shutdown preserves an unowned endpoint and its persisted state", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-unowned-"); + const socketPath = path.join(sessionDir, "fake-broker.sock"); + const server = net.createServer(); + await new Promise((resolve) => server.listen(socketPath, resolve)); + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + if (fs.existsSync(socketPath)) { + fs.unlinkSync(socketPath); + } + }); + + const session = { + endpoint: `unix:${socketPath}`, + pid: null, + pidFile: null, + logFile: null, + sessionDir + }; + saveBrokerSession(workspace, session); + + await assert.rejects( + shutdownBrokerSession(workspace, { + timeoutMs: 40, + killProcess: terminateProcessTree + }), + /ownership could not be verified/i + ); + + assert.equal(fs.existsSync(socketPath), true); + assert.deepEqual(loadBrokerSession(workspace), session); +}); + +test("shutdown never removes artifacts outside a private broker session", async () => { + const workspace = makeTempDir(); + const externalDir = makeTempDir("not-a-broker-session-"); + const externalFile = path.join(externalDir, "preserve.txt"); + fs.writeFileSync(externalFile, "preserve me\n"); + saveBrokerSession(workspace, { + endpoint: `unix:${path.join(externalDir, "missing.sock")}`, + pid: null, + pidFile: externalFile, + logFile: null, + sessionDir: externalDir + }); + + const outcome = await shutdownBrokerSession(workspace); + + assert.equal(outcome.exited, true); + assert.equal(fs.readFileSync(externalFile, "utf8"), "preserve me\n"); + assert.equal(loadBrokerSession(workspace), null); +}); + +test("shutdown never follows a private-session symlink", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const externalDir = makeTempDir("not-a-broker-session-"); + const externalPidFile = path.join(externalDir, "broker.pid"); + const sessionLink = makeTempDir("cxc-"); + fs.writeFileSync(externalPidFile, "preserve me\n"); + fs.rmdirSync(sessionLink); + fs.symlinkSync(externalDir, sessionLink, "dir"); + t.after(() => fs.unlinkSync(sessionLink)); + + saveBrokerSession(workspace, { + endpoint: `unix:${path.join(sessionLink, "missing.sock")}`, + pid: null, + pidFile: path.join(sessionLink, "broker.pid"), + logFile: null, + sessionDir: sessionLink + }); + + const outcome = await shutdownBrokerSession(workspace); + + assert.equal(outcome.exited, true); + assert.equal(fs.readFileSync(externalPidFile, "utf8"), "preserve me\n"); + assert.equal(loadBrokerSession(workspace), null); +}); + +async function spawnLegacyBroker(t, { socketPath, pidFile, endpoint, cwdArg, ackShutdown = true }) { + const child = spawn( + process.execPath, + [ + "-e", + `const fs = require("node:fs"); + const net = require("node:net"); + const socketPath = ${JSON.stringify(socketPath)}; + const pidFile = ${JSON.stringify(pidFile)}; + const ackShutdown = ${JSON.stringify(ackShutdown)}; + fs.writeFileSync(pidFile, String(process.pid)); + const server = net.createServer((socket) => { + socket.setEncoding("utf8"); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk; + if (!buffer.includes("\\n")) return; + const request = JSON.parse(buffer.slice(0, buffer.indexOf("\\n"))); + if (request.method !== "broker/shutdown" || !ackShutdown) return; + socket.end(JSON.stringify({ id: request.id, result: {} }) + "\\n", () => { + server.close(() => process.exit(0)); + }); + }); + }); + server.listen(socketPath, () => process.stdout.write("ready\\n"));`, + "serve", + "--endpoint", + endpoint, + "--cwd", + cwdArg, + "--pid-file", + pidFile + ], + { detached: true, stdio: ["ignore", "pipe", "ignore"] } + ); + await new Promise((resolve, reject) => { + child.stdout.on("data", (chunk) => { + if (String(chunk).includes("ready")) { + resolve(); + } + }); + child.once("error", reject); + child.once("exit", () => reject(new Error("legacy broker exited before binding"))); + }); + t.after(() => { + if (isProcessTreeRunning(child.pid)) { + terminateProcessTree(child.pid); + } + }); + return child; +} + +test("shutdown retires a live tokenless broker left by the previous version", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-legacy-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const pidFile = path.join(sessionDir, "broker.pid"); + const endpoint = `unix:${socketPath}`; + const child = await spawnLegacyBroker(t, { socketPath, pidFile, endpoint, cwdArg: workspace }); + + const session = { + endpoint, + pid: child.pid, + pidFile, + logFile: null, + sessionDir + }; + saveBrokerSession(workspace, session); + + const outcome = await shutdownBrokerSession(workspace, { + timeoutMs: 500, + intervalMs: 10, + killProcess: terminateProcessTree + }); + + assert.equal(outcome.exited, true); + assert.equal(outcome.forced, false); + assert.equal(loadBrokerSession(workspace), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("shutdown retires a legacy broker launched from a different workspace path", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-legacy-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const pidFile = path.join(sessionDir, "broker.pid"); + const endpoint = `unix:${socketPath}`; + // The previous plugin version may have launched the broker with a --cwd + // that addressed this workspace through another path (subdirectory, + // symlink). Ownership must not depend on the current invocation path. + const originalCwd = path.join(workspace, "nested", "launch-dir"); + fs.mkdirSync(originalCwd, { recursive: true }); + const child = await spawnLegacyBroker(t, { socketPath, pidFile, endpoint, cwdArg: originalCwd }); + + saveBrokerSession(workspace, { + endpoint, + pid: child.pid, + pidFile, + logFile: null, + sessionDir + }); + + const outcome = await shutdownBrokerSession(workspace, { + timeoutMs: 500, + intervalMs: 10, + killProcess: terminateProcessTree + }); + + assert.equal(outcome.exited, true); + assert.equal(loadBrokerSession(workspace), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("shutdown force-kills a verified legacy broker that ignores shutdown requests", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-legacy-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const pidFile = path.join(sessionDir, "broker.pid"); + const endpoint = `unix:${socketPath}`; + const child = await spawnLegacyBroker(t, { + socketPath, + pidFile, + endpoint, + cwdArg: workspace, + ackShutdown: false + }); + + saveBrokerSession(workspace, { + endpoint, + pid: child.pid, + pidFile, + logFile: null, + sessionDir + }); + + const outcome = await shutdownBrokerSession(workspace, { + timeoutMs: 500, + intervalMs: 10, + killProcess: terminateProcessTree + }); + + assert.equal(outcome.exited, true); + assert.equal(outcome.forced, true); + assert.equal(loadBrokerSession(workspace), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("shutdown does not retire an authenticated broker whose persisted token was lost", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const authenticatedSession = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + assert.ok(authenticatedSession?.instanceToken); + t.after(async () => { + saveBrokerSession(workspace, authenticatedSession); + await shutdownBrokerSession(workspace, { + killProcess: terminateProcessTree + }); + }); + + const { instanceToken: _lostToken, ...tokenlessState } = authenticatedSession; + saveBrokerSession(workspace, tokenlessState); + + await assert.rejects( + shutdownBrokerSession(workspace, { + timeoutMs: 200, + killProcess: terminateProcessTree + }), + /rejected shutdown identity/i + ); + + assert.equal(isProcessTreeRunning(authenticatedSession.pid), true); + assert.deepEqual(loadBrokerSession(workspace), tokenlessState); +}); + +test("broker metadata, pid, and log files are private", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const session = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + assert.ok(session); + + const persistedStateFile = path.join(resolveStateDir(workspace), "broker.json"); + + assert.equal(fs.statSync(session.sessionDir).mode & 0o777, 0o700); + assert.equal(fs.statSync(session.pidFile).mode & 0o777, 0o600); + assert.equal(fs.statSync(session.logFile).mode & 0o777, 0o600); + assert.equal(fs.statSync(persistedStateFile).mode & 0o777, 0o600); + + await shutdownBrokerSession(workspace, { + killProcess: terminateProcessTree + }); +}); + +async function spawnSocketHolder(socketPath) { + const child = spawn( + process.execPath, + [ + "-e", + `const net = require("node:net"); + const server = net.createServer(); + server.listen(${JSON.stringify(socketPath)}, () => process.stdout.write("ready\\n")); + setInterval(() => {}, 60000);` + ], + { stdio: ["ignore", "pipe", "ignore"] } + ); + await new Promise((resolve, reject) => { + child.stdout.on("data", (chunk) => { + if (String(chunk).includes("ready")) { + resolve(); + } + }); + child.once("error", reject); + child.once("exit", () => reject(new Error("socket holder exited before binding"))); + }); + return child; +} + +test("shutdown reclaims the socket of an owned broker that died without acking", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-stale-"); + const socketPath = path.join(sessionDir, "broker.sock"); + + // Killing the holder with SIGKILL skips its cleanup, so the socket file stays + // on disk with nothing listening — exactly what a crashed broker leaves behind. + const holder = await spawnSocketHolder(socketPath); + const deadPid = holder.pid; + holder.kill("SIGKILL"); + await new Promise((resolve) => holder.once("exit", resolve)); + assert.equal(fs.existsSync(socketPath), true, "stale socket should survive the kill"); + + const session = { + endpoint: `unix:${socketPath}`, + pid: deadPid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "stale-instance-token" + }; + saveBrokerSession(workspace, session); + + const outcome = await shutdownBrokerSession(workspace, { + timeoutMs: 40, + killProcess: terminateProcessTree + }); + + assert.equal(outcome.exited, true); + assert.equal(outcome.reclaimedStaleEndpoint, true); + assert.equal(fs.existsSync(socketPath), false, "stale socket should be unlinked"); + assert.equal(loadBrokerSession(workspace), null); +}); + +test("ensureBrokerSession recovers from a stale socket instead of failing", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-stale-ensure-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const holder = await spawnSocketHolder(socketPath); + const deadPid = holder.pid; + holder.kill("SIGKILL"); + await new Promise((resolve) => holder.once("exit", resolve)); + + saveBrokerSession(workspace, { + endpoint: `unix:${socketPath}`, + pid: deadPid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "stale-instance-token" + }); + + const session = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + t.after(async () => { + if (session) { + await shutdownBrokerSession(workspace, { killProcess: terminateProcessTree }); + } + }); + + assert.ok(session, "a stale socket must not block the replacement broker"); + assert.notEqual(session.endpoint, `unix:${socketPath}`); +}); + +test("ensureBrokerSession fails cleanly when the broker process cannot spawn", async () => { + const workspace = makeTempDir(); + // spawn() launches the broker with `cwd` as the child's working directory. + // Deleting it after makeTempDir() reproduces a workspace that vanishes + // between the caller's check and the spawn: Node cannot fail synchronously + // here, it emits an asynchronous "error" event (ENOENT) instead. + fs.rmSync(workspace, { recursive: true, force: true }); + + const before = new Set(fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("cxc-"))); + + await assert.rejects(() => ensureBrokerSession(workspace, { env: process.env }), /failed to spawn/); + + assert.equal( + loadBrokerSession(workspace), + null, + "a spawn failure must never persist a tokenized session with pid: null, or shutdown wedges permanently" + ); + + const after = new Set(fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("cxc-"))); + const leaked = [...after].filter((name) => !before.has(name)); + assert.deepEqual(leaked, [], "the failed attempt's temporary session directory must not be left behind"); +}); + +test("shutdown still refuses to unlink an endpoint someone is listening on", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-live-listener-"); + const socketPath = path.join(sessionDir, "broker.sock"); + + // Dead PID, but a live listener owns the path: reclaiming would delete the + // socket of an unrelated process, so the shutdown must keep failing. + const holder = await spawnSocketHolder(socketPath); + const stalePid = holder.pid; + holder.kill("SIGKILL"); + await new Promise((resolve) => holder.once("exit", resolve)); + if (fs.existsSync(socketPath)) { + fs.unlinkSync(socketPath); + } + // A listener that accepts but never answers — the shape of a hung broker. Its + // sockets must be tracked: server.close() waits for every accepted connection + // to end, and the shutdown request leaves one open. + const accepted = new Set(); + const server = net.createServer((socket) => { + accepted.add(socket); + socket.on("close", () => accepted.delete(socket)); + }); + await new Promise((resolve) => server.listen(socketPath, resolve)); + t.after(async () => { + for (const socket of accepted) { + socket.destroy(); + } + await new Promise((resolve) => server.close(resolve)); + if (fs.existsSync(socketPath)) { + fs.unlinkSync(socketPath); + } + }); + + const session = { + endpoint: `unix:${socketPath}`, + pid: stalePid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "stale-instance-token" + }; + saveBrokerSession(workspace, session); + + await assert.rejects( + shutdownBrokerSession(workspace, { + timeoutMs: 40, + killProcess: terminateProcessTree + }), + /ownership could not be verified/i + ); + + assert.equal(fs.existsSync(socketPath), true); + assert.deepEqual(loadBrokerSession(workspace), session); +}); + +test("a refused endpoint alone does not justify reclaiming while the owner lives", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-live-owner-"); + const socketPath = path.join(sessionDir, "broker.sock"); + + // Live owner, refused endpoint: the socket file exists but nothing listens on + // it, which on its own looks exactly like the stale case. The recorded PID is + // still running, so the shutdown must not unlink anything. + const holder = await spawnSocketHolder(socketPath); + t.after(() => { + holder.kill("SIGKILL"); + }); + fs.unlinkSync(socketPath); + fs.writeFileSync(socketPath, ""); + + const session = { + endpoint: `unix:${socketPath}`, + pid: holder.pid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "live-owner-token" + }; + saveBrokerSession(workspace, session); + + await assert.rejects( + shutdownBrokerSession(workspace, { + timeoutMs: 40, + killProcess: () => {} + }), + /ownership could not be verified|did not exit/i + ); + + assert.equal(fs.existsSync(socketPath), true); + assert.deepEqual(loadBrokerSession(workspace), session); +}); + +test("an endpoint outside the plugin session directory is never reclaimed", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-outside-"); + const elsewhere = makeTempDir("not-a-session-dir-"); + const socketPath = path.join(elsewhere, "broker.sock"); + + const holder = await spawnSocketHolder(socketPath); + const deadPid = holder.pid; + holder.kill("SIGKILL"); + await new Promise((resolve) => holder.once("exit", resolve)); + assert.equal(fs.existsSync(socketPath), true); + + // Everything else looks reclaimable — dead PID, refused connect — but the + // endpoint does not live under the session directory we created. + const session = { + endpoint: `unix:${socketPath}`, + pid: deadPid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "outside-token" + }; + saveBrokerSession(workspace, session); + + await assert.rejects( + shutdownBrokerSession(workspace, { + timeoutMs: 40, + killProcess: terminateProcessTree + }), + /ownership could not be verified/i + ); + + assert.equal(fs.existsSync(socketPath), true, "foreign socket must survive"); + fs.unlinkSync(socketPath); +}); + +test("shutdown aborts instead of silently skipping an unreadable pid file", async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-unreadable-pid-"); + const pidFile = path.join(sessionDir, "broker.pid"); + // A directory at the pid file path makes the read fail with EISDIR instead + // of ENOENT, exercising the "any other failure must abort" branch of + // resolveBrokerPid() rather than the "file genuinely absent" branch. + fs.mkdirSync(pidFile); + + // Deliberately no instanceToken: this session can never be cleaned up + // through shutdownBrokerSession() (the unreadable pid file always aborts + // it, by design), so it must stay invisible to the global owned-broker + // teardown in tests/helpers.mjs, which only acts on sessions carrying one. + const session = { + endpoint: `unix:${path.join(sessionDir, "broker.sock")}`, + pid: null, + pidFile, + logFile: null, + sessionDir + }; + saveBrokerSession(workspace, session); + + await assert.rejects( + shutdownBrokerSession(workspace, { timeoutMs: 40 }), + (error) => error?.code === "EISDIR" + ); + + assert.deepEqual(loadBrokerSession(workspace), session); +}); + +test("ensureBrokerSession tears down a spawned child when persisting the session fails", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(stateDir, { recursive: true }); + // Occupies the broker.json path with a directory so saveBrokerSession's + // rename-into-place fails with EISDIR only after the child has already + // spawned successfully. + const brokerStateFile = path.join(stateDir, "broker.json"); + fs.mkdirSync(brokerStateFile); + + const killedPids = []; + const killProcess = (pid) => { + killedPids.push(pid); + return terminateProcessTree(pid); + }; + + // Capture the exact session directory ensureBrokerSessionLocked creates + // via the same injection point other tests use, instead of diffing + // os.tmpdir() -- concurrent test files also create cxc-* directories, so a + // tmpdir-wide before/after snapshot races with them. + let capturedSessionDir = null; + const createBrokerEndpointSpy = (sessionDir, platform) => { + capturedSessionDir = sessionDir; + return createBrokerEndpoint(sessionDir, platform); + }; + + await assert.rejects( + ensureBrokerSession(workspace, { env: buildEnv(binDir), killProcess, createBrokerEndpoint: createBrokerEndpointSpy }) + ); + + assert.equal(killedPids.length, 1, "the spawned child must be killed exactly once"); + assert.ok(Number.isSafeInteger(killedPids[0]) && killedPids[0] > 0); + + assert.ok(capturedSessionDir, "ensureBrokerSession must have created a session directory"); + assert.equal( + fs.existsSync(capturedSessionDir), + false, + "the session directory of the failed persist must not be left behind" + ); + + assert.equal(loadBrokerSession(workspace), null); + assert.equal(fs.existsSync(brokerStateFile), true, "the blocking directory itself is left untouched"); + assert.equal(fs.statSync(brokerStateFile).isDirectory(), true); +}); + +test("shutdown treats a reused PID as exited once the recorded identity stops matching", { skip: process.platform === "win32" }, async (t) => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-reused-pid-"); + const socketPath = path.join(sessionDir, "broker.sock"); + + // A detached process is its own process-group leader (unlike this test's + // own process under `node --test`, which is not) -- exactly the shape of + // an impostor that reused a dead broker's PID and became an unrelated new + // group leader. Its real identity can never equal the bogus one recorded + // below, so liveness must be resolved from session.processIdentity rather + // than group membership alone, which would otherwise read it as running. + const dummy = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + detached: true, + stdio: "ignore" + }); + await new Promise((resolve, reject) => { + dummy.once("spawn", resolve); + dummy.once("error", reject); + }); + t.after(async () => { + if (!isProcessTreeRunning(dummy.pid)) { + return; + } + terminateProcessTree(dummy.pid); + let exited = await waitForProcessExit(dummy.pid, { timeoutMs: 2000 }); + if (!exited) { + try { + process.kill(-dummy.pid, "SIGKILL"); + } catch { + // Already gone between the check above and here. + } + exited = await waitForProcessExit(dummy.pid, { timeoutMs: 2000 }); + } + assert.equal(exited, true, "cleanup must confirm the test process exited"); + }); + + const session = { + endpoint: `unix:${socketPath}`, + pid: dummy.pid, + pidFile: null, + logFile: null, + sessionDir, + instanceToken: "reused-pid-token", + processIdentity: "reused-pid-identity-mismatch" + }; + saveBrokerSession(workspace, session); + + const outcome = await shutdownBrokerSession(workspace, { timeoutMs: 40 }); + + assert.equal(outcome.found, true); + assert.equal(outcome.exited, true); + assert.equal(outcome.forced, false); + assert.equal(loadBrokerSession(workspace), null); +}); + +test("ensureBrokerSession preserves session artifacts when the killed child does not actually exit", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(stateDir, { recursive: true }); + // Same EISDIR trigger as the sibling test above, just paired here with a + // killProcess that does not actually stop the child, to exercise the + // "kill was denied or too slow" branch of the failed-persist unwind. + const brokerStateFile = path.join(stateDir, "broker.json"); + fs.mkdirSync(brokerStateFile); + + let killedPid = null; + const killProcess = (pid) => { + killedPid = pid; + // Deliberately a no-op: simulates a kill signal that was denied or did + // not land in time. The child stays alive. + }; + + // Capture the exact session directory via the same injection point other + // tests use, instead of diffing os.tmpdir() -- concurrent test files also + // create cxc-* directories, so a tmpdir-wide before/after snapshot races + // with them. + let sessionDir = null; + const createBrokerEndpointSpy = (dir, platform) => { + sessionDir = dir; + return createBrokerEndpoint(dir, platform); + }; + + try { + await assert.rejects( + ensureBrokerSession(workspace, { + env: buildEnv(binDir), + killProcess, + timeoutMs: 150, + createBrokerEndpoint: createBrokerEndpointSpy + }), + /did not exit.*artifacts were preserved/i + ); + + assert.ok(Number.isSafeInteger(killedPid) && killedPid > 0, "killProcess must have been invoked with the child pid"); + assert.equal(isProcessTreeRunning(killedPid), true, "the child must still be alive for this branch to be exercised"); + + assert.ok(sessionDir, "ensureBrokerSession must have created a session directory"); + assert.equal(fs.existsSync(sessionDir), true, "the session directory must not be torn down while the child is still alive"); + + assert.equal(fs.existsSync(brokerStateFile), true, "the blocking directory itself is left untouched"); + assert.equal(fs.statSync(brokerStateFile).isDirectory(), true); + } finally { + // The injected killProcess deliberately did nothing, so the real child is + // still running: terminate and reap it for real before the test exits, + // then remove the preserved session directory ourselves so this test's + // deliberate leak does not confuse the leak-detection windows of other + // tests running in the same tmpdir. Only rmSync after confirming exit -- + // otherwise a still-alive process could recreate files under sessionDir + // after we remove it. + if (killedPid) { + terminateProcessTree(killedPid); + let exited = await waitForProcessExit(killedPid, { timeoutMs: 2000 }); + if (!exited) { + try { + process.kill(-killedPid, "SIGKILL"); + } catch { + // Already gone between the check above and here. + } + exited = await waitForProcessExit(killedPid, { timeoutMs: 2000 }); + } + assert.equal(exited, true, "cleanup must confirm the test process exited"); + } + if (sessionDir) { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + } +}); + +// Spawns a detached leader (its own process group) that binds the broker +// endpoint socket and spawns a non-detached grandchild -- inheriting the +// leader's process group, per standard POSIX fork/exec -- so the grandchild +// keeps the group alive long after the leader itself is killed. This is the +// exact "abandoned orphaned tree" shape CHANGE 1/2 exist to reclaim from: a +// dead leader whose launch-token proof died with it, but whose process group +// still has live, unrelated-looking members. +async function spawnOrphanLeader(socketPath) { + const leader = spawn( + process.execPath, + [ + "-e", + `const net = require("node:net"); + const { spawn } = require("node:child_process"); + const socketPath = ${JSON.stringify(socketPath)}; + const grandchild = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"], { stdio: "ignore" }); + grandchild.unref(); + grandchild.once("spawn", () => { + const server = net.createServer(); + server.listen(socketPath, () => { + process.stdout.write("ready " + grandchild.pid + "\\n"); + }); + }); + setInterval(() => {}, 60000);` + ], + { detached: true, stdio: ["ignore", "pipe", "ignore"] } + ); + const grandchildPid = await new Promise((resolve, reject) => { + leader.stdout.on("data", (chunk) => { + const match = String(chunk).match(/ready (\d+)/); + if (match) { + resolve(Number(match[1])); + } + }); + leader.once("error", reject); + leader.once("exit", () => reject(new Error("orphan leader exited before binding"))); + }); + return { leader, grandchildPid }; +} + +// Builds the abandoned-orphan fixture and persists a session for it. Passing +// no instanceToken produces a legacy session (the key is omitted entirely, +// matching how genuinely legacy sessions are built elsewhere in this file). +async function setupOrphanedSession(workspace, { instanceToken } = {}) { + const sessionDir = makeTempDir("cxc-orphan-"); + const socketPath = path.join(sessionDir, "broker.sock"); + const pidFile = path.join(sessionDir, "broker.pid"); + + const { leader, grandchildPid } = await spawnOrphanLeader(socketPath); + const leaderPid = leader.pid; + fs.writeFileSync(pidFile, String(leaderPid)); + + // Captured while the leader is alive -- this is what + // ensureBrokerSessionLocked would have persisted at spawn time. + const recordedIdentity = getProcessIdentity(leaderPid); + assert.ok(recordedIdentity, "expected a recordable identity while the leader is alive"); + + const session = { + endpoint: `unix:${socketPath}`, + pid: leaderPid, + pidFile, + logFile: null, + sessionDir, + processIdentity: recordedIdentity, + ...(instanceToken !== undefined ? { instanceToken } : {}) + }; + saveBrokerSession(workspace, session); + + // Kill only the leader: SIGKILL skips its own cleanup, so the socket file + // stays on disk with nothing listening, exactly like a crashed broker. + leader.kill("SIGKILL"); + const deadline = Date.now() + 5000; + while (Date.now() < deadline && isProcessRunning(leaderPid)) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + assert.equal(isProcessRunning(leaderPid), false, "the leader should have exited by now"); + assert.equal(fs.existsSync(socketPath), true, "the stale socket must survive the leader's death"); + assert.equal(isProcessTreeRunning(leaderPid), true, "the grandchild must keep the process group alive"); + + return { sessionDir, socketPath, pidFile, session, leaderPid, grandchildPid }; +} + +async function cleanupOrphanGroup(leaderPid, grandchildPid) { + try { + process.kill(-leaderPid, "SIGTERM"); + } catch { + // Group may already be gone. + } + // The grandchild, not the (already dead) leader, is what a plain + // isProcessTreeRunning(grandchildPid) cannot see -- it inherited the + // leader's pgid rather than being a group leader itself -- so poll its own + // liveness directly (tree: false) instead. + let exited = await waitForProcessExit(grandchildPid, { timeoutMs: 2000, tree: false }); + if (!exited) { + try { + process.kill(-leaderPid, "SIGKILL"); + } catch { + // Already gone between the check above and here. + } + exited = await waitForProcessExit(grandchildPid, { timeoutMs: 2000, tree: false }); + } + assert.equal(exited, true, "cleanup must confirm the test process exited"); +} + +test("shutdown reclaims a tokened session whose dead leader left an orphaned group", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const { socketPath, pidFile, leaderPid, grandchildPid } = await setupOrphanedSession(workspace, { + instanceToken: "orphan-leader-token" + }); + + let killProcessCalled = false; + const killProcess = () => { + killProcessCalled = true; + }; + + try { + const outcome = await shutdownBrokerSession(workspace, { timeoutMs: 40, killProcess }); + + assert.equal(killProcessCalled, false, "an abandoned orphaned tree must never be signaled"); + assert.deepEqual(outcome, { found: true, exited: true, forced: false, reclaimedStaleEndpoint: true }); + assert.equal(isProcessRunning(grandchildPid), true, "the grandchild must be left running untouched"); + assert.equal(fs.existsSync(socketPath), false, "the stale socket must be removed"); + assert.equal(fs.existsSync(pidFile), false, "the pid file must be removed"); + assert.equal(loadBrokerSession(workspace), null); + } finally { + await cleanupOrphanGroup(leaderPid, grandchildPid); + } +}); + +test("ensureBrokerSession recovers after reclaiming an orphaned group", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const { + sessionDir: oldSessionDir, + socketPath: oldSocketPath, + session: oldSession, + leaderPid, + grandchildPid + } = await setupOrphanedSession(workspace, { instanceToken: "orphan-leader-token" }); + + try { + const session = await ensureBrokerSession(workspace, { env: buildEnv(binDir) }); + + assert.ok(session, "ensureBrokerSession must recover instead of wedging on the orphaned tree"); + assert.notEqual(session.pid, leaderPid); + assert.notEqual(session.sessionDir, oldSessionDir); + assert.notEqual(session.endpoint, oldSession.endpoint); + + const probe = net.createConnection({ path: session.endpoint.slice("unix:".length) }); + await new Promise((resolve, reject) => { + probe.once("connect", resolve); + probe.once("error", reject); + }); + probe.destroy(); + + await shutdownBrokerSession(workspace, { killProcess: terminateProcessTree }); + assert.equal(loadBrokerSession(workspace), null); + + assert.equal(fs.existsSync(oldSocketPath), false, "the reclaimed stale socket must have been removed"); + assert.equal(isProcessRunning(grandchildPid), true, "the orphaned grandchild must be left running throughout"); + } finally { + await cleanupOrphanGroup(leaderPid, grandchildPid); + } +}); + +test("a legacy orphaned group still preserves state", { skip: process.platform === "win32" }, async () => { + const workspace = makeTempDir(); + // No instanceToken: pins the CHANGE 1/2 relaxation strictly to the tokened + // path -- a legacy session hitting the identical dead-leader/live-group + // shape must still refuse and preserve state. + const { socketPath, leaderPid, grandchildPid } = await setupOrphanedSession(workspace); + + try { + await assert.rejects( + shutdownBrokerSession(workspace, { timeoutMs: 40, killProcess: terminateProcessTree }), + /ownership could not be verified/i + ); + + assert.equal(fs.existsSync(socketPath), true, "the socket must be preserved for a legacy orphaned group"); + assert.ok(loadBrokerSession(workspace), "broker.json must be preserved for a legacy orphaned group"); + } finally { + await cleanupOrphanGroup(leaderPid, grandchildPid); + } +}); diff --git a/tests/helpers.mjs b/tests/helpers.mjs index d6981197a..ee5ea89d4 100644 --- a/tests/helpers.mjs +++ b/tests/helpers.mjs @@ -3,11 +3,50 @@ import os from "node:os"; import path from "node:path"; import process from "node:process"; import { spawnSync } from "node:child_process"; +import { after } from "node:test"; + +import { + loadBrokerSession, + shutdownBrokerSession +} from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; + +const trackedTempDirs = []; export function makeTempDir(prefix = "codex-plugin-test-") { - return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + trackedTempDirs.push(dir); + return dir; } +// Global teardown: any broker a test started (directly or lazily) and did not +// tear down is shut down here, so the suite never leaves broker/app-server +// processes behind — even when a test fails or returns early. +after(async () => { + const failures = []; + for (const dir of trackedTempDirs) { + const session = loadBrokerSession(dir); + // A fixture may deliberately persist an arbitrary endpoint. Only sessions + // with the per-process secret introduced by ensureBrokerSession are ours. + if (!session?.instanceToken) { + continue; + } + try { + await shutdownBrokerSession(dir, { + killProcess: terminateProcessTree, + timeoutMs: 1000, + intervalMs: 25 + }); + } catch (error) { + failures.push(error); + } + } + + if (failures.length > 0) { + throw new AggregateError(failures, "Owned broker teardown did not finish."); + } +}); + export function writeExecutable(filePath, source) { fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o755 }); } diff --git a/tests/locking.test.mjs b/tests/locking.test.mjs new file mode 100644 index 000000000..37eba6e4b --- /dev/null +++ b/tests/locking.test.mjs @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { spawn } from "node:child_process"; +import test from "node:test"; + +import { + acquireLock, + acquireLockSync, + releaseLock +} from "../plugins/codex/scripts/lib/locking.mjs"; +import { makeTempDir } from "./helpers.mjs"; + +test("lock records its owner privately and release removes only that generation", () => { + const lockDir = path.join(makeTempDir(), "state.lock"); + const first = acquireLockSync(lockDir); + const owner = JSON.parse(fs.readFileSync(first.ownerFile, "utf8")); + + assert.equal(owner.pid, process.pid); + assert.equal(owner.token, first.token); + if (process.platform !== "win32") { + assert.equal(fs.statSync(lockDir).mode & 0o777, 0o700); + assert.equal(fs.statSync(first.ownerFile).mode & 0o777, 0o600); + } + + assert.equal(releaseLock(first), true); + const successor = acquireLockSync(lockDir); + + assert.equal(releaseLock(first), false); + assert.equal(fs.existsSync(successor.ownerFile), true); + assert.throws( + () => acquireLockSync(lockDir, { timeoutMs: 30, retryDelayMs: 5 }), + /Timed out waiting for lock/ + ); + + assert.equal(releaseLock(successor), true); +}); + +test("lock immediately reclaims an owner process that exited", async () => { + const lockDir = path.join(makeTempDir(), "state.lock"); + const lockingModuleUrl = new URL( + "../plugins/codex/scripts/lib/locking.mjs", + import.meta.url + ).href; + + await new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { acquireLockSync } from ${JSON.stringify(lockingModuleUrl)}; + acquireLockSync(${JSON.stringify(lockDir)}); + process.stdout.write("owned");` + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("exit", (code, signal) => { + if (code === 0 && stdout === "owned") { + resolve(); + return; + } + reject(new Error(`owner failed (code=${code} signal=${signal}): ${stderr}`)); + }); + }); + + const startedAt = Date.now(); + const successor = await acquireLock(lockDir, { + timeoutMs: 500, + staleMs: 30000, + retryDelayMs: 5 + }); + assert.ok(Date.now() - startedAt < 500, "dead owner should be reclaimed before stale timeout"); + releaseLock(successor); +}); + +test("unknown lock contents fail closed", () => { + const lockDir = path.join(makeTempDir(), "state.lock"); + fs.mkdirSync(lockDir); + fs.writeFileSync(path.join(lockDir, "unexpected"), "do not delete\n"); + + assert.throws( + () => acquireLockSync(lockDir, { timeoutMs: 30, staleMs: 0, retryDelayMs: 5 }), + /Timed out waiting for lock/ + ); + assert.equal(fs.readFileSync(path.join(lockDir, "unexpected"), "utf8"), "do not delete\n"); +}); + +test("malformed lock owners fail closed", () => { + const owners = [ + { pid: 1.5, token: "fractional-pid", processIdentity: null }, + { pid: process.pid, token: "object-identity", processIdentity: {} } + ]; + + for (const owner of owners) { + const lockDir = path.join(makeTempDir(), "state.lock"); + const ownerFile = path.join(lockDir, `owner-${owner.token}.json`); + fs.mkdirSync(lockDir); + fs.writeFileSync(ownerFile, `${JSON.stringify(owner)}\n`); + + assert.throws( + () => acquireLockSync(lockDir, { timeoutMs: 30, staleMs: 0, retryDelayMs: 5 }), + /Timed out waiting for lock/ + ); + assert.equal(fs.existsSync(ownerFile), true); + } +}); + +test("stale legacy lock without process identity does not follow a reused PID forever", () => { + const lockDir = path.join(makeTempDir(), "state.lock"); + const token = "legacy-owner"; + fs.mkdirSync(lockDir, { mode: 0o700 }); + fs.writeFileSync( + path.join(lockDir, `owner-${token}.json`), + `${JSON.stringify({ + pid: process.pid, + token, + createdAt: Date.now() - 60000, + processIdentity: null + })}\n`, + { mode: 0o600 } + ); + const oldTime = new Date(Date.now() - 60000); + fs.utimesSync(lockDir, oldTime, oldTime); + + const successor = acquireLockSync(lockDir, { + timeoutMs: 200, + staleMs: 30000, + retryDelayMs: 5, + isProcessRunning: () => true + }); + + assert.notEqual(successor.token, token); + assert.equal(releaseLock(successor), true); +}); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 80e0715b0..87da53291 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,156 @@ +import process from "node:process"; import test from "node:test"; import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; -import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { + getProcessIdentity, + isProcessRunning, + isProcessTreeRunning, + processHasLaunchSequence, + runCommand, + runCommandChecked, + terminateProcessTree, + waitForProcessExit +} from "../plugins/codex/scripts/lib/process.mjs"; + +const SELF_TERMINATING_SCRIPT = "process.kill(process.pid, 'SIGTERM'); setInterval(() => {}, 1000);"; + +test("runCommand reports a signal-terminated process as a failure", { skip: process.platform === "win32" }, () => { + const result = runCommand(process.execPath, ["-e", SELF_TERMINATING_SCRIPT]); + + assert.equal(result.signal, "SIGTERM"); + assert.equal(result.status, null); +}); + +test("runCommandChecked throws when the process dies from a signal", { skip: process.platform === "win32" }, () => { + assert.throws( + () => runCommandChecked(process.execPath, ["-e", SELF_TERMINATING_SCRIPT]), + /signal=SIGTERM/ + ); +}); + +test("Linux zombie processes are treated as exited even when signal 0 succeeds", () => { + const running = isProcessRunning(1234, { + platform: "linux", + killImpl() {}, + readProcessStat() { + return { state: "Z", startTime: "42" }; + } + }); + + assert.equal(running, false); +}); + +test("Linux processes with unreadable /proc metadata stay running", () => { + const running = isProcessRunning(1234, { + platform: "linux", + identity: "42", + killImpl() {}, + readProcessStat() { + return null; + } + }); + + assert.equal(running, true); +}); + +test("macOS process identity uses the process start time", () => { + let captured = null; + const identity = getProcessIdentity(1234, { + platform: "darwin", + runCommandImpl(command, args) { + captured = { command, args }; + return { + command, + args, + status: 0, + signal: null, + stdout: "Fri Jul 25 01:02:03 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.deepEqual(captured, { + command: "ps", + args: ["-ww", "-p", "1234", "-o", "lstart="] + }); + assert.equal(identity, "Fri Jul 25 01:02:03 2026"); +}); + +test("Windows process identity uses PowerShell start-time ticks", () => { + let capturedCommand = null; + const identity = getProcessIdentity(1234, { + platform: "win32", + runCommandImpl(command, args) { + capturedCommand = { command, args }; + return { + command, + args, + status: 0, + signal: null, + stdout: "638890021230000000", + stderr: "", + error: null + }; + } + }); + + assert.equal(capturedCommand.command, "powershell.exe"); + assert.match(capturedCommand.args.at(-1), /Get-Process -Id 1234/); + assert.equal(identity, "638890021230000000"); +}); + +test("non-Linux process identity distinguishes a reused PID", () => { + const running = isProcessRunning(1234, { + platform: "darwin", + identity: "original-start", + killImpl() {}, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "replacement-start\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(running, false); +}); + +test("process launch sequence fallback requires arguments in order", () => { + const runCommandImpl = (command, args) => ({ + command, + args, + status: 0, + signal: null, + stdout: "node app-server-broker.mjs serve --endpoint pipe:broker --cwd /workspace --pid-file /tmp/broker.pid", + stderr: "", + error: null + }); + + assert.equal( + processHasLaunchSequence( + 1234, + ["serve", "--endpoint", "pipe:broker", "--cwd", "/workspace", "--pid-file", "/tmp/broker.pid"], + { platform: "darwin", runCommandImpl } + ), + true + ); + assert.equal( + processHasLaunchSequence(1234, ["--cwd", "/workspace", "--endpoint", "pipe:broker"], { + platform: "darwin", + runCommandImpl + }), + false + ); +}); test("terminateProcessTree uses taskkill on Windows", () => { let captured = null; @@ -53,3 +202,67 @@ test("terminateProcessTree treats missing Windows processes as already stopped", assert.equal(outcome.result.status, 128); assert.match(outcome.result.stdout, /not found/i); }); + +test("a dead group leader with a surviving descendant still counts as a running tree", { skip: process.platform === "win32" }, async () => { + // The leader spawns a grandchild without detaching it: an un-detached + // child inherits its parent's process group (standard POSIX fork/exec + // behavior), so the grandchild keeps the leader's original group alive + // long after the leader itself has exited and been reaped. + const leaderScript = ` + const { spawn } = require("node:child_process"); + const grandchild = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"], { stdio: "ignore" }); + grandchild.unref(); + setTimeout(() => process.exit(0), 200); + `; + const leader = spawn(process.execPath, ["-e", leaderScript], { + detached: true, + stdio: "ignore" + }); + await new Promise((resolve, reject) => { + leader.once("spawn", resolve); + leader.once("error", reject); + }); + const leaderPid = leader.pid; + + try { + const recordedIdentity = getProcessIdentity(leaderPid); + assert.ok(recordedIdentity, "expected a recordable identity while the leader is alive"); + + // Poll for the leader's own exit instead of a fixed sleep: its script + // exits itself after ~200ms, but scheduling jitter under test-suite load + // must not make this flaky. + const deadline = Date.now() + 5000; + while (Date.now() < deadline && isProcessRunning(leaderPid)) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + assert.equal(isProcessRunning(leaderPid), false, "the leader should have exited and been reaped by now"); + + // The leader is gone, but the grandchild it left behind is still in the + // leader's original process group. A live descendant must still read as + // a running tree, matched against the identity recorded while the + // leader itself was alive. + assert.equal( + isProcessTreeRunning(leaderPid, { identity: recordedIdentity }), + true, + "a surviving descendant must keep the recorded process group alive" + ); + } finally { + // The leader is already dead; -leaderPid still addresses the process + // group as long as at least one member (the grandchild) survives. + try { + process.kill(-leaderPid, "SIGTERM"); + } catch { + // Group may already be gone. + } + let exited = await waitForProcessExit(leaderPid, { timeoutMs: 2000 }); + if (!exited) { + try { + process.kill(-leaderPid, "SIGKILL"); + } catch { + // Already gone between the check above and here. + } + exited = await waitForProcessExit(leaderPid, { timeoutMs: 2000 }); + } + assert.equal(exited, true, "cleanup must confirm the test process exited"); + } +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 8f276835b..1ef209d8a 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -1923,6 +1923,31 @@ test("session end fully cleans up jobs for the ending session", async (t) => { assert.equal(otherJob.logFile, otherSessionLog); }); +test("session end reports the underlying cleanup failure", (t) => { + const repo = makeTempDir(); + const brokerStateFile = path.join(resolveStateDir(repo), "broker.json"); + t.after(() => fs.rmSync(brokerStateFile, { force: true })); + saveBrokerSession(repo, { + endpoint: "unsupported:broker", + pid: process.pid, + pidFile: null, + logFile: null, + sessionDir: null, + instanceToken: "instance-token-1234567890" + }); + + const result = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + input: JSON.stringify({ + hook_event_name: "SessionEnd", + cwd: repo + }) + }); + + assert.equal(result.status, 1); + assert.match(result.stderr, /Unsupported broker endpoint: unsupported:broker/); +}); + test("stop hook runs a stop-time review task and blocks on findings when the review gate is enabled", () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -2233,6 +2258,16 @@ test("status reports shared session runtime when a lazy broker is active", () => assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /Session runtime: shared session/); + + const cleanup = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: buildEnv(binDir), + input: JSON.stringify({ + hook_event_name: "SessionEnd", + cwd: repo + }) + }); + assert.equal(cleanup.status, 0, cleanup.stderr); }); test("setup and status honor --cwd when reading shared session runtime", () => { diff --git a/tests/state.test.mjs b/tests/state.test.mjs index 0f8f57cea..6bfae30de 100644 --- a/tests/state.test.mjs +++ b/tests/state.test.mjs @@ -1,11 +1,24 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import process from "node:process"; import test from "node:test"; import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { makeTempDir } from "./helpers.mjs"; -import { resolveJobFile, resolveJobLogFile, resolveStateDir, resolveStateFile, saveState } from "../plugins/codex/scripts/lib/state.mjs"; +import { + listJobs, + resolveJobFile, + resolveJobLogFile, + resolveJobsDir, + resolveStateDir, + resolveStateFile, + saveState, + writeJobFile +} from "../plugins/codex/scripts/lib/state.mjs"; +import { createJobLogFile } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; test("resolveStateDir uses a temp-backed per-workspace directory", () => { const workspace = makeTempDir(); @@ -40,6 +53,23 @@ test("resolveStateDir uses CLAUDE_PLUGIN_DATA when it is provided", () => { } }); +test("state, job, and log artifacts are private", { skip: process.platform === "win32" }, () => { + const workspace = makeTempDir(); + saveState(workspace, { + version: 1, + config: { stopReviewGate: false }, + jobs: [] + }); + const jobFile = writeJobFile(workspace, "private-job", { prompt: "sensitive prompt" }); + const logFile = createJobLogFile(workspace, "private-job", "Private Job"); + + assert.equal(fs.statSync(resolveStateDir(workspace)).mode & 0o777, 0o700); + assert.equal(fs.statSync(resolveJobsDir(workspace)).mode & 0o777, 0o700); + assert.equal(fs.statSync(resolveStateFile(workspace)).mode & 0o777, 0o600); + assert.equal(fs.statSync(jobFile).mode & 0o777, 0o600); + assert.equal(fs.statSync(logFile).mode & 0o777, 0o600); +}); + test("saveState prunes dropped job artifacts when indexed jobs exceed the cap", () => { const workspace = makeTempDir(); const stateFile = resolveStateFile(workspace); @@ -103,3 +133,47 @@ test("saveState prunes dropped job artifacts when indexed jobs exceed the cap", .sort() ); }); + +test("concurrent upsertJob calls from separate processes do not lose updates", async () => { + const workspace = makeTempDir(); + const stateModuleUrl = pathToFileURL( + path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "plugins", "codex", "scripts", "lib", "state.mjs") + ).href; + const jobsPerWorker = 15; + + const spawnWorker = (prefix) => + new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { upsertJob } from ${JSON.stringify(stateModuleUrl)}; + for (let index = 0; index < ${jobsPerWorker}; index++) { + upsertJob(${JSON.stringify(workspace)}, { id: ${JSON.stringify(prefix)} + "-" + index }); + }` + ], + { stdio: ["ignore", "ignore", "pipe"] } + ); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("exit", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + reject(new Error(`worker ${prefix} failed (code=${code} signal=${signal}): ${stderr}`)); + }); + }); + + await Promise.all([spawnWorker("left"), spawnWorker("right")]); + + const jobIds = new Set(listJobs(workspace).map((job) => job.id)); + for (let index = 0; index < jobsPerWorker; index++) { + assert.equal(jobIds.has(`left-${index}`), true, `missing left-${index}`); + assert.equal(jobIds.has(`right-${index}`), true, `missing right-${index}`); + } +});