diff --git a/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.test.ts b/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.test.ts index b456484588..6e781c8561 100644 --- a/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.test.ts +++ b/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { Env, SandboxInstance } from '../../types.js'; import type { SessionMetadata } from '../../persistence/session-metadata.js'; import { WrapperClient } from '../../kilo/wrapper-client.js'; +import { WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS } from '../../shared/protocol.js'; import { WRAPPER_VERSION } from '../../shared/wrapper-version.js'; import type { EnsureWrapperRequest } from '../protocol.js'; import { CloudflareAgentSandbox } from './cloudflare-agent-sandbox.js'; @@ -541,6 +542,36 @@ describe('CloudflareAgentSandbox', () => { expect(exec).toHaveBeenCalledWith(expect.stringContaining('--agent-session agent_cloudflare')); }); + it('allows the wrapper final-log deadline before force stopping', async () => { + const observedProcess = { + id: 'wrapper-target', + command: + 'WRAPPER_PORT=5000 kilocode-wrapper --agent-session agent_cloudflare --wrapper-instance-id instance_1 --wrapper-instance-generation 2', + status: 'running', + }; + const listProcesses = vi.fn().mockResolvedValue([observedProcess]); + const sleep = vi.fn().mockResolvedValue(undefined); + const sandbox = new CloudflareAgentSandbox({} as Env, metadata(), { + resolveSandbox: () => + ({ + listProcesses, + exec: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '' }), + }) as unknown as SandboxInstance, + sleep, + }); + + await expect( + sandbox.stopWrappers({ + target: { kind: 'instance', instance: { instanceId: 'instance_1', instanceGeneration: 2 } }, + attemptId: 'attempt_graceful_flush', + reason: 'readiness-failed', + }) + ).resolves.toMatchObject({ status: 'still-present' }); + + const gracefulStopWindowMs = sleep.mock.calls.reduce((total, [delayMs]) => total + delayMs, 0); + expect(gracefulStopWindowMs).toBeGreaterThan(WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS); + }); + it('returns still-present when targeted forceful cleanup remains observable', async () => { const observedProcess = { id: 'wrapper-target', diff --git a/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.ts b/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.ts index f0e7ce6b43..526446f0ba 100644 --- a/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.ts +++ b/services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.ts @@ -38,6 +38,7 @@ import { } from '../../sandbox-timeout-logging.js'; import { SANDBOX_WORKSPACE_PROBE_TIMEOUT_MESSAGE } from '../../sandbox-recovery.js'; import { withTimeout } from '@kilocode/worker-utils'; +import { WRAPPER_GRACEFUL_STOP_TIMEOUT_MS } from '../../shared/protocol.js'; import { WRAPPER_VERSION } from '../../shared/wrapper-version.js'; import { ExecutionError } from '../../execution/errors.js'; import { @@ -47,7 +48,15 @@ import { } from '../../workspace-errors.js'; const PREPARE_WORKSPACE_TIMEOUT_MS = 10 * 60 * 1000; -const DEFAULT_STOP_OBSERVATION_DELAYS_MS = [100, 500, 1_000]; +const EARLY_STOP_OBSERVATION_DELAYS_MS = [100, 500, 1_000, 2_000, 5_000, 10_000]; +const EARLY_STOP_OBSERVATION_WINDOW_MS = EARLY_STOP_OBSERVATION_DELAYS_MS.reduce( + (total, delayMs) => total + delayMs, + 0 +); +const DEFAULT_STOP_OBSERVATION_DELAYS_MS = [ + ...EARLY_STOP_OBSERVATION_DELAYS_MS, + WRAPPER_GRACEFUL_STOP_TIMEOUT_MS - EARLY_STOP_OBSERVATION_WINDOW_MS, +]; function withWorkspacePreparationTimeout(operation: Promise, step: string): Promise { return withTimeout( diff --git a/services/cloud-agent-next/src/shared/protocol.ts b/services/cloud-agent-next/src/shared/protocol.ts index 466ca6110b..490d1ec161 100644 --- a/services/cloud-agent-next/src/shared/protocol.ts +++ b/services/cloud-agent-next/src/shared/protocol.ts @@ -51,6 +51,9 @@ export type IngestEvent = { data: unknown; }; +export const WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS = 32_000; +export const WRAPPER_GRACEFUL_STOP_TIMEOUT_MS = WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS + 3_000; + export const WrapperTerminalFailureCodes = ['payment_required', 'model_missing'] as const; export type WrapperTerminalFailureCode = (typeof WrapperTerminalFailureCodes)[number]; diff --git a/services/cloud-agent-next/wrapper/src/lifecycle.test.ts b/services/cloud-agent-next/wrapper/src/lifecycle.test.ts index 9ad2363204..979000abb7 100644 --- a/services/cloud-agent-next/wrapper/src/lifecycle.test.ts +++ b/services/cloud-agent-next/wrapper/src/lifecycle.test.ts @@ -3,6 +3,7 @@ import { WrapperState } from './state'; import { createLifecycleManager } from './lifecycle'; import type { IngestEvent } from '../../src/shared/protocol'; import type { WrapperKiloClient } from './kilo-api'; +import type { LogUploader } from './log-uploader'; const sessionContext = { kiloSessionId: 'kilo_sess_test', @@ -19,6 +20,53 @@ function wait(ms: number): Promise { } describe('wrapper lifecycle drain races', () => { + it('flushes immutable logs before emitting completion', async () => { + const state = new WrapperState(); + const events: IngestEvent[] = []; + const order: string[] = []; + let releaseFlush: (() => void) | undefined; + const flushGate = new Promise(resolve => { + releaseFlush = resolve; + }); + const uploader: LogUploader = { + start: () => {}, + uploadNow: async () => {}, + flushNow: async () => { + order.push('flush-started'); + await flushGate; + order.push('flush-finished'); + }, + stop: () => {}, + }; + state.bindSession(sessionContext); + state.setSendToIngestFn(event => { + events.push(event); + if (event.streamEventType === 'complete') order.push('complete'); + }); + state.setLogUploader(uploader); + const lifecycle = createLifecycleManager( + { workspacePath: '/tmp' }, + { + state, + kiloClient: {} as WrapperKiloClient, + closeConnections: async () => {}, + isConnected: () => true, + reconnectEventSubscription: () => {}, + } + ); + + lifecycle.triggerDrainAndClose(); + while (!order.includes('flush-started')) await wait(1); + expect(events.map(event => event.streamEventType)).not.toContain('complete'); + + const release = releaseFlush; + if (!release) throw new Error('Expected final log flush to start'); + release(); + while (!order.includes('complete')) await wait(1); + + expect(order).toEqual(['flush-started', 'flush-finished', 'complete']); + }); + it('clears aborted state when activity cancels an aborted drain', async () => { const state = new WrapperState(); const events: IngestEvent[] = []; diff --git a/services/cloud-agent-next/wrapper/src/lifecycle.ts b/services/cloud-agent-next/wrapper/src/lifecycle.ts index 1f1ce3eb73..51ec57e71d 100644 --- a/services/cloud-agent-next/wrapper/src/lifecycle.ts +++ b/services/cloud-agent-next/wrapper/src/lifecycle.ts @@ -173,13 +173,12 @@ export function createLifecycleManager( const uploader = state.logUploader; if (uploader) { try { - await uploader.uploadNow(); + await uploader.flushNow(); } catch (error) { logToFile( `final log upload failed: ${error instanceof Error ? error.message : String(error)}` ); } - uploader.stop(); } } finally { const currentSession = state.currentSession; diff --git a/services/cloud-agent-next/wrapper/src/log-uploader.test.ts b/services/cloud-agent-next/wrapper/src/log-uploader.test.ts new file mode 100644 index 0000000000..199a8bf76e --- /dev/null +++ b/services/cloud-agent-next/wrapper/src/log-uploader.test.ts @@ -0,0 +1,162 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createLogUploader } from './log-uploader'; +import { getKiloImportDiagnosticPath } from './utils'; + +function asFetch( + fn: (...args: Parameters) => ReturnType +): typeof fetch { + return Object.assign(fn, { preconnect: fetch.preconnect }); +} + +function requestUrl(input: string | URL | Request): string { + if (typeof input === 'string') return input; + return input instanceof URL ? input.href : input.url; +} + +describe('createLogUploader', () => { + let tmpDir: string; + let originalFetch: typeof globalThis.fetch; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'log-uploader-test-')); + originalFetch = globalThis.fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('uploads the import failure diagnostic in the authenticated R2 log archive', async () => { + const cliLogDir = path.join(tmpDir, 'cli-logs'); + const wrapperLogPath = path.join(tmpDir, 'wrapper.log'); + const diagnosticPath = getKiloImportDiagnosticPath(wrapperLogPath); + fs.mkdirSync(cliLogDir); + fs.writeFileSync(path.join(cliLogDir, 'kilo.log'), 'cli log'); + fs.writeFileSync(wrapperLogPath, 'wrapper log'); + fs.writeFileSync(diagnosticPath, '{"version":1,"stderr":{"text":"schema failure"}}'); + + const requests: Array<{ url: string; authorization: string | null; archive: Uint8Array }> = []; + globalThis.fetch = asFetch(async (input, init) => { + const body = init?.body; + if (!(body instanceof ReadableStream)) throw new Error('Expected streaming archive body'); + requests.push({ + url: requestUrl(input), + authorization: new Headers(init?.headers).get('Authorization'), + archive: new Uint8Array(await new Response(body).arrayBuffer()), + }); + return new Response(null, { status: 204 }); + }); + + const uploader = createLogUploader({ + workerBaseUrl: 'https://worker.example', + sessionId: 'agent_test', + executionId: 'session', + userId: 'user_test', + workerAuthToken: 'worker-token', + cliLogDir, + wrapperLogPath, + }); + await uploader.uploadNow(); + + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe( + 'https://worker.example/sessions/user_test/agent_test/logs/session/logs.tar.gz' + ); + expect(requests[0]?.authorization).toBe('Bearer worker-token'); + + const archivePath = path.join(tmpDir, 'logs.tar.gz'); + const extractDir = path.join(tmpDir, 'extracted'); + fs.writeFileSync(archivePath, requests[0]?.archive ?? new Uint8Array()); + fs.mkdirSync(extractDir); + const extraction = Bun.spawnSync(['tar', 'xzf', archivePath, '-C', extractDir], { + stdout: 'pipe', + stderr: 'pipe', + }); + expect(extraction.exitCode).toBe(0); + expect(fs.readFileSync(path.join(extractDir, path.basename(diagnosticPath)), 'utf8')).toContain( + 'schema failure' + ); + }); + + it('serializes final uploads into one bounded final archive key', async () => { + const cliLogDir = path.join(tmpDir, 'cli-logs'); + const wrapperLogPath = path.join(tmpDir, 'wrapper.log'); + const diagnosticPath = getKiloImportDiagnosticPath(wrapperLogPath); + fs.mkdirSync(cliLogDir); + fs.writeFileSync(wrapperLogPath, 'wrapper log'); + + const archives: Uint8Array[] = []; + const requestUrls: string[] = []; + const uploadOrder: string[] = []; + let firstUploadSignal: AbortSignal | undefined; + let releaseFirstResponse: (() => void) | undefined; + const firstResponseGate = new Promise(resolve => { + releaseFirstResponse = resolve; + }); + globalThis.fetch = asFetch(async (input, init) => { + const body = init?.body; + if (!(body instanceof ReadableStream)) throw new Error('Expected streaming archive body'); + requestUrls.push(requestUrl(input)); + const requestNumber = requestUrls.length; + uploadOrder.push(`start-${requestNumber}`); + archives.push(new Uint8Array(await new Response(body).arrayBuffer())); + if (requestNumber === 1) { + firstUploadSignal = init?.signal ?? undefined; + await firstResponseGate; + } + uploadOrder.push(`settle-${requestNumber}`); + return new Response(null, { status: 204 }); + }); + + const uploader = createLogUploader({ + workerBaseUrl: 'https://worker.example', + sessionId: 'agent_test', + executionId: 'session', + userId: 'user_test', + workerAuthToken: 'worker-token', + cliLogDir, + wrapperLogPath, + }); + const periodicUpload = uploader.uploadNow(); + while (requestUrls.length === 0) await Bun.sleep(1); + + fs.writeFileSync(diagnosticPath, '{"version":1,"stderr":{"text":"late failure"}}'); + const finalUpload = uploader.flushNow(); + await Promise.resolve(); + await Promise.resolve(); + const firstUploadWasAborted = firstUploadSignal?.aborted ?? false; + const finalUploadStartedBeforeFirstSettled = requestUrls.length > 1; + const release = releaseFirstResponse; + if (!release) throw new Error('First upload did not start'); + release(); + await Promise.all([periodicUpload, finalUpload]); + + expect(firstUploadWasAborted).toBe(false); + expect(finalUploadStartedBeforeFirstSettled).toBe(false); + expect(uploadOrder).toEqual(['start-1', 'settle-1', 'start-2', 'settle-2']); + expect(requestUrls[0]).toContain('/logs/session/logs.tar.gz'); + expect(requestUrls[1]).toContain('/logs/session-final/logs.tar.gz'); + + await uploader.flushNow(); + + expect(requestUrls[2]).toBe(requestUrls[1]); + expect(new Set(requestUrls).size).toBe(2); + expect(archives).toHaveLength(3); + const archivePath = path.join(tmpDir, 'final-logs.tar.gz'); + const extractDir = path.join(tmpDir, 'final-extracted'); + fs.writeFileSync(archivePath, archives[1] ?? new Uint8Array()); + fs.mkdirSync(extractDir); + const extraction = Bun.spawnSync(['tar', 'xzf', archivePath, '-C', extractDir], { + stdout: 'pipe', + stderr: 'pipe', + }); + expect(extraction.exitCode).toBe(0); + expect(fs.readFileSync(path.join(extractDir, path.basename(diagnosticPath)), 'utf8')).toContain( + 'late failure' + ); + }); +}); diff --git a/services/cloud-agent-next/wrapper/src/log-uploader.ts b/services/cloud-agent-next/wrapper/src/log-uploader.ts index 5f415fa37e..5ba38b2f87 100644 --- a/services/cloud-agent-next/wrapper/src/log-uploader.ts +++ b/services/cloud-agent-next/wrapper/src/log-uploader.ts @@ -1,7 +1,7 @@ import { existsSync } from 'node:fs'; import { basename, dirname } from 'node:path'; import { spawn } from 'node:child_process'; -import { logToFile } from './utils.js'; +import { getKiloImportDiagnosticPath, logToFile } from './utils.js'; type LogUploaderOpts = { workerBaseUrl: string; @@ -17,6 +17,7 @@ type LogUploaderOpts = { export type LogUploader = { start: (intervalMs?: number) => void; uploadNow: () => Promise; + flushNow: () => Promise; stop: () => void; }; @@ -81,21 +82,20 @@ function createTarStream(paths: Array): TarStream | undefined { export function createLogUploader(opts: LogUploaderOpts): LogUploader { let intervalId: ReturnType | undefined; - let isUploading = false; - - async function uploadNow(): Promise { - if (isUploading) return; - isUploading = true; - const tar = createTarStream([opts.cliLogDir, opts.wrapperLogPath]); - if (!tar) { - isUploading = false; - return; - } + let uploadQueue = Promise.resolve(); + + async function performUpload(executionId: string): Promise { + const tar = createTarStream([ + opts.cliLogDir, + opts.wrapperLogPath, + getKiloImportDiagnosticPath(opts.wrapperLogPath), + ]); + if (!tar) return; const abort = new AbortController(); const timer = setTimeout(() => abort.abort(), UPLOAD_TIMEOUT_MS); try { - const url = `${opts.workerBaseUrl}/sessions/${encodeURIComponent(opts.userId)}/${encodeURIComponent(opts.sessionId)}/logs/${encodeURIComponent(opts.executionId)}/logs.tar.gz`; + const url = `${opts.workerBaseUrl}/sessions/${encodeURIComponent(opts.userId)}/${encodeURIComponent(opts.sessionId)}/logs/${encodeURIComponent(executionId)}/logs.tar.gz`; const response = await fetch(url, { method: 'PUT', headers: { Authorization: `Bearer ${opts.workerAuthToken}` }, @@ -116,10 +116,19 @@ export function createLogUploader(opts: LogUploaderOpts): LogUploader { } finally { clearTimeout(timer); tar.kill(); - isUploading = false; } } + function enqueueUpload(executionId: string): Promise { + const upload = uploadQueue.then(() => performUpload(executionId)); + uploadQueue = upload.catch(() => {}); + return upload; + } + + function uploadNow(): Promise { + return enqueueUpload(opts.executionId); + } + function start(intervalMs = 30_000): void { stop(); intervalId = setInterval(() => { @@ -134,5 +143,10 @@ export function createLogUploader(opts: LogUploaderOpts): LogUploader { } } - return { start, uploadNow, stop }; + function flushNow(): Promise { + stop(); + return enqueueUpload(`${opts.executionId}-final`); + } + + return { start, uploadNow, flushNow, stop }; } diff --git a/services/cloud-agent-next/wrapper/src/main.ts b/services/cloud-agent-next/wrapper/src/main.ts index 33e499403a..958d250668 100644 --- a/services/cloud-agent-next/wrapper/src/main.ts +++ b/services/cloud-agent-next/wrapper/src/main.ts @@ -12,7 +12,11 @@ */ import { createKilo } from '@kilocode/sdk'; -import { SESSION_ID_RE } from '../../src/shared/protocol.js'; +import { + SESSION_ID_RE, + WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS, + WRAPPER_GRACEFUL_STOP_TIMEOUT_MS, +} from '../../src/shared/protocol.js'; import { WRAPPER_VERSION } from '../../src/shared/wrapper-version.js'; import { WrapperState } from './state.js'; import { createWrapperKiloClient, type WrapperKiloClient } from './kilo-api.js'; @@ -21,7 +25,7 @@ import { createLifecycleManager } from './lifecycle.js'; import { bindSessionContext, createServer } from './server.js'; import { openKiloGlobalFeed } from './global-feed.js'; import { createGlobalFeedManager, type SessionBoundFeedPolicy } from './global-feed-manager.js'; -import { logToFile } from './utils.js'; +import { applyMaterializedEnvironment, logToFile } from './utils.js'; import { kiloServerBootstrapError, kiloServerStartupError, @@ -41,8 +45,8 @@ import { // Constants // --------------------------------------------------------------------------- -/** Grace period before force exit during shutdown (20 seconds) */ -const SHUTDOWN_TIMEOUT_MS = 20_000; +/** Grace period before force exit during shutdown. */ +const SHUTDOWN_TIMEOUT_MS = WRAPPER_GRACEFUL_STOP_TIMEOUT_MS + 30_000; /** Timeout for createKilo() server startup */ const KILO_STARTUP_TIMEOUT_MS = 30_000; @@ -508,9 +512,9 @@ async function main() { async function updateRuntimeEnvironment(env: Record): Promise { const environmentChanged = Object.entries(env).some( - ([name, value]) => process.env[name] !== value + ([name, value]) => name !== 'WRAPPER_LOG_PATH' && process.env[name] !== value ); - Object.assign(process.env, env); + applyMaterializedEnvironment(env); if (runtimeWorkspacePath && (environmentChanged || !kiloClient)) { await startKiloRuntime(runtimeWorkspacePath, kiloSessionId || undefined, true); } @@ -693,6 +697,8 @@ async function main() { logToFile(`shutdown signal: ${signal}`); console.error(`Received ${signal}, shutting down...`); + const uploader = state.logUploader; + uploader?.stop(); // Force exit after timeout setTimeout(() => { @@ -711,6 +717,14 @@ async function main() { timestamp: new Date().toISOString(), }); + // Start the final upload before startup cleanup so the sandbox grace window is reserved for logs. + const finalLogUpload = + uploader && + Promise.race([ + uploader.flushNow().catch(() => {}), + new Promise(resolve => setTimeout(resolve, WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS)), + ]); + workspaceBootstrapController.abort(); const workspaceBootstraps = [...activeWorkspaceBootstraps]; const runtimeStartups = [...activeRuntimeStartups]; @@ -728,12 +742,7 @@ async function main() { globalFeedManager.close(); // Best-effort final log upload - const uploader = state.logUploader; - if (uploader) { - const uploadTimeout = new Promise(resolve => setTimeout(resolve, 5_000)); - await Promise.race([uploader.uploadNow().catch(() => {}), uploadTimeout]); - uploader.stop(); - } + await finalLogUpload; // Abort kilo session if running const session = state.currentSession; @@ -778,9 +787,10 @@ async function main() { const uploader = state.logUploader; if (uploader) { - const timeout = new Promise(resolve => setTimeout(resolve, 5_000)); - void Promise.race([uploader.uploadNow().catch(() => {}), timeout]).finally(() => { - uploader.stop(); + const timeout = new Promise(resolve => + setTimeout(resolve, WRAPPER_FINAL_LOG_UPLOAD_TIMEOUT_MS) + ); + void Promise.race([uploader.flushNow().catch(() => {}), timeout]).finally(() => { process.exit(1); }); } else { diff --git a/services/cloud-agent-next/wrapper/src/restore-session.test.ts b/services/cloud-agent-next/wrapper/src/restore-session.test.ts index cfb7395349..a0fb497f8c 100644 --- a/services/cloud-agent-next/wrapper/src/restore-session.test.ts +++ b/services/cloud-agent-next/wrapper/src/restore-session.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; import { restoreSession, extractDiffs } from './restore-session'; +import { getKiloImportDiagnosticPath } from './utils'; // --------------------------------------------------------------------------- // Helpers @@ -80,6 +81,12 @@ function writeMockKilo(binDir: string, exitCode: number): void { fs.writeFileSync(kiloPath, script, { mode: 0o755 }); } +function writeMockKiloFailure(binDir: string, stdout: string, stderr: string): void { + const script = `#!/usr/bin/env node\nprocess.stdout.write(${JSON.stringify(stdout)});\nprocess.stderr.write(${JSON.stringify(stderr)});\nprocess.exitCode = 1;\n`; + const kiloPath = path.join(binDir, 'kilo'); + fs.writeFileSync(kiloPath, script, { mode: 0o755 }); +} + function writeSlowMockKilo(binDir: string): void { const script = '#!/bin/sh\nsleep 1\nexit 0\n'; const kiloPath = path.join(binDir, 'kilo'); @@ -99,6 +106,36 @@ function writeSignalIgnoringDescendantMockKilo(binDir: string, descendantMarker: fs.writeFileSync(kiloPath, script, { mode: 0o755 }); } +// Keep credential URLs assembled from non-URL fragments so secret scanners do not flag fixtures. +const URL_CREDENTIAL_SEPARATOR = String.fromCharCode(64); +const FAKE_DATABASE_URL = [ + 'postgres://database-user', + ':database-password', + URL_CREDENTIAL_SEPARATOR, + 'database.example/app', +].join(''); +const FAKE_SENTRY_DSN = [ + 'sentry://sentry-secret', + URL_CREDENTIAL_SEPARATOR, + 'sentry.example/project', +].join(''); +const FAKE_REDIS_URL = [ + 'redis://:redis-password', + URL_CREDENTIAL_SEPARATOR, + 'redis.example/0', +].join(''); +const FAKE_TOKEN_ONLY_URL = [ + 'https://token-only', + URL_CREDENTIAL_SEPARATOR, + 'example.com/path', +].join(''); +const FAKE_CREDENTIAL_URL = [ + 'https://user:url-secret', + URL_CREDENTIAL_SEPARATOR, + 'example.com/repo.git?', + 'X-Amz-Signature=signed-secret', +].join(''); + // --------------------------------------------------------------------------- // Test suite // --------------------------------------------------------------------------- @@ -123,14 +160,22 @@ describe('restoreSession', () => { writeMockKilo(binDir, 0); savedEnv = { + DATABASE_URL: process.env.DATABASE_URL, + DB_PASS: process.env.DB_PASS, KILO_SESSION_INGEST_URL: process.env.KILO_SESSION_INGEST_URL, + SENTRY_DSN: process.env.SENTRY_DSN, KILOCODE_TOKEN: process.env.KILOCODE_TOKEN, KILOCODE_TOKEN_FILE: process.env.KILOCODE_TOKEN_FILE, + WRAPPER_LOG_PATH: process.env.WRAPPER_LOG_PATH, PATH: process.env.PATH, }; + process.env.DATABASE_URL = FAKE_DATABASE_URL; + process.env.DB_PASS = 'db-secret'; process.env.KILO_SESSION_INGEST_URL = 'http://localhost:9999'; + process.env.SENTRY_DSN = FAKE_SENTRY_DSN; process.env.KILOCODE_TOKEN = 'test-token'; + process.env.WRAPPER_LOG_PATH = path.join(tmpDir, 'wrapper.log'); delete process.env.KILOCODE_TOKEN_FILE; process.env.PATH = `${binDir}:${process.env.PATH}`; @@ -337,6 +382,127 @@ describe('restoreSession', () => { // ---- Import failures ---- + it('persists useful redacted import output outside public and wrapper logs', async () => { + mockFetchOk(makeSnapshot([])); + writeMockKiloFailure( + binDir, + `import validation failed at messages[4].parts[2]\nKILOCODE_TOKEN=test-token\nSECRET_VALUE=env-secret\nDB_PASS=db-secret\nDATABASE_URL=${FAKE_DATABASE_URL}\nbare test-\u001b[31mtoken\nbare test-\u0000token\nbare opaque-secret\n`, + `invalid discriminator value\nAuthorization: Bearer bearer-secret\nX-Api-Key: api-secret\nCookie: session=cookie-secret\n{"token":"prefix\\"json-secret"}\nSENTRY_DSN=${FAKE_SENTRY_DSN}\n${FAKE_REDIS_URL}\n${FAKE_TOKEN_ONLY_URL}\n${FAKE_CREDENTIAL_URL}\n-----BEGIN PRIVATE KEY-----\nprivate-key-secret\n-----END PRIVATE KEY-----\n-----BEGIN OPENSSH PRIVATE KEY-----\nunterminated-private-key-secret\n` + ); + + const result = await restoreSession(SESSION_ID, workspace, undefined, { + sensitiveValues: ['opaque-secret'], + }); + + expect(result.ok).toBe(false); + const wrapperLogPath = process.env.WRAPPER_LOG_PATH; + if (!wrapperLogPath) throw new Error('Expected WRAPPER_LOG_PATH'); + const diagnosticPath = getKiloImportDiagnosticPath(wrapperLogPath); + const diagnostic = JSON.parse(fs.readFileSync(diagnosticPath, 'utf8')) as { + version: number; + kiloSessionId: string; + process: { exitCode: number }; + stdout: { text: string; truncated: boolean }; + stderr: { text: string; truncated: boolean }; + }; + expect(diagnostic).toMatchObject({ + version: 1, + kiloSessionId: SESSION_ID, + process: { exitCode: 1 }, + stdout: { truncated: false }, + stderr: { truncated: false }, + }); + expect(diagnostic.stdout.text).toContain('import validation failed at messages[4].parts[2]'); + expect(diagnostic.stdout.text.match(/^bare \[REDACTED\]$/gm)).toHaveLength(3); + expect(diagnostic.stdout.text).not.toContain('[31m'); + expect(diagnostic.stdout.text).not.toContain('\u0000'); + expect(diagnostic.stderr.text).toContain('invalid discriminator value'); + const serializedDiagnostic = JSON.stringify(diagnostic); + expect(serializedDiagnostic).not.toContain('\\u0000'); + for (const credential of [ + 'test-token', + 'env-secret', + 'db-secret', + 'bearer-secret', + 'api-secret', + 'json-secret', + 'cookie-secret', + 'database-password', + 'opaque-secret', + 'sentry-secret', + 'redis-password', + 'token-only', + 'url-secret', + 'signed-secret', + 'private-key-secret', + 'unterminated-private-key-secret', + ]) { + expect(serializedDiagnostic).not.toContain(credential); + } + expect(fs.statSync(diagnosticPath).mode & 0o777).toBe(0o600); + + const projectedResult = JSON.stringify(result); + const wrapperLog = fs.readFileSync(wrapperLogPath, 'utf8'); + for (const privateOutput of [ + 'messages[4].parts[2]', + 'invalid discriminator value', + 'test-token', + 'env-secret', + 'db-secret', + 'bearer-secret', + 'api-secret', + 'json-secret', + 'cookie-secret', + 'database-password', + 'opaque-secret', + 'sentry-secret', + 'redis-password', + 'token-only', + 'url-secret', + 'signed-secret', + 'private-key-secret', + 'unterminated-private-key-secret', + ]) { + expect(projectedResult).not.toContain(privateOutput); + expect(wrapperLog).not.toContain(privateOutput); + } + }); + + it('does not follow a pre-created temporary diagnostic symlink', async () => { + mockFetchOk(makeSnapshot([])); + writeMockKiloFailure(binDir, '', 'schema failure'); + const diagnosticPath = getKiloImportDiagnosticPath(process.env.WRAPPER_LOG_PATH); + const predictableTemporaryPath = `${diagnosticPath}.${process.pid}.tmp`; + const symlinkTarget = path.join(tmpDir, 'symlink-target'); + fs.writeFileSync(symlinkTarget, 'sentinel', { mode: 0o644 }); + fs.symlinkSync(symlinkTarget, predictableTemporaryPath); + + const result = await restoreSession(SESSION_ID, workspace); + + expect(result.ok).toBe(false); + expect(fs.lstatSync(diagnosticPath).isSymbolicLink()).toBe(false); + expect(fs.readFileSync(symlinkTarget, 'utf8')).toBe('sentinel'); + expect(fs.statSync(diagnosticPath).mode & 0o777).toBe(0o600); + }); + + it('omits truncated import output from the restricted diagnostic', async () => { + mockFetchOk(makeSnapshot([])); + writeMockKiloFailure(binDir, `${'x'.repeat(70 * 1_024)}latest-error`, ''); + + const result = await restoreSession(SESSION_ID, workspace); + + expect(result.ok).toBe(false); + const diagnosticPath = getKiloImportDiagnosticPath(process.env.WRAPPER_LOG_PATH); + const diagnostic = JSON.parse(fs.readFileSync(diagnosticPath, 'utf8')) as { + stdout: { text: string; truncated: boolean }; + }; + expect(diagnostic.stdout).toEqual({ + text: '[omitted: captured output was truncated]', + truncated: true, + }); + expect(fs.statSync(diagnosticPath).size).toBeLessThan(2_048); + }); + it('returns import error when kilo import fails', async () => { const snapshot = makeSnapshot([{ file: 'src/index.ts', after: 'content', status: 'modified' }]); mockFetchOk(snapshot); @@ -402,7 +568,12 @@ describe('restoreSession', () => { if (!result.ok) { expect(result.step).toBe('import'); expect(result.error).toContain('kilo import failed'); + expect(result.detail).toContain('signal SIGTERM'); } + const diagnostic = JSON.parse( + fs.readFileSync(getKiloImportDiagnosticPath(process.env.WRAPPER_LOG_PATH), 'utf8') + ) as { process: { signal?: string } }; + expect(diagnostic.process.signal).toBe('SIGTERM'); }); it('kills signal-ignoring descendants after a kilo import timeout grace period', async () => { @@ -446,6 +617,7 @@ describe('restoreSession', () => { imported: true, diffs: { applied: 2, skipped: 0, total: 2 }, }); + expect(fs.existsSync(getKiloImportDiagnosticPath(process.env.WRAPPER_LOG_PATH))).toBe(false); // Verify modified file was written const created = fs.readFileSync(path.join(workspace, 'src/index.ts'), 'utf-8'); @@ -455,6 +627,22 @@ describe('restoreSession', () => { expect(fs.existsSync(path.join(workspace, 'old-file.txt'))).toBe(false); }); + it('removes stale import failure diagnostics after a successful import', async () => { + mockFetchOk(makeSnapshot([])); + const diagnosticPath = getKiloImportDiagnosticPath(process.env.WRAPPER_LOG_PATH); + fs.writeFileSync(diagnosticPath, '{"version":1,"stderr":{"text":"old failure"}}'); + + const result = await restoreSession(SESSION_ID, workspace); + + expect(result).toEqual({ + ok: true, + downloaded: true, + imported: true, + diffs: { applied: 0, skipped: 0, total: 0 }, + }); + expect(fs.existsSync(diagnosticPath)).toBe(false); + }); + it('prefers top-level patch session diffs over legacy message summaries', async () => { const repo = path.join(tmpDir, 'repo'); fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); diff --git a/services/cloud-agent-next/wrapper/src/restore-session.ts b/services/cloud-agent-next/wrapper/src/restore-session.ts index eefca48f72..a5d2db3aa6 100644 --- a/services/cloud-agent-next/wrapper/src/restore-session.ts +++ b/services/cloud-agent-next/wrapper/src/restore-session.ts @@ -4,9 +4,11 @@ import path from 'node:path'; import type { WorkspaceFailureSubtype } from '../../src/shared/wrapper-bootstrap.js'; import { createSafeProcessDiagnostic, + getKiloImportDiagnosticPath, isTimeoutTermination, logToFile, runProcess, + type ExecResult, } from './utils.js'; // --------------------------------------------------------------------------- @@ -39,10 +41,19 @@ type SnapshotDiff = { export type RestoreSessionOptions = { importTimeoutMs?: number; importTerminationGraceMs?: number; + diagnosticPath?: string; + sensitiveValues?: string[]; signal?: AbortSignal; }; const KILO_IMPORT_TIMEOUT_MS = 120_000; +const OMITTED_TRUNCATED_OUTPUT = '[omitted: captured output was truncated]'; +const REDACTED = '[REDACTED]'; +const ANSI_ESCAPE_PATTERN = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, 'g'); +const SENSITIVE_ENV_NAME = + /(?:AUTH|COOKIE|CREDENTIAL|DSN|KEY|PASS|PRIVATE|SECRET|SIGNATURE|TOKEN|URL)/i; +const SENSITIVE_OUTPUT_LINE = + /(?:\b(?:authorization|cookie|set-cookie|x-api-key)\s*:|(?:^|[\s"'{}[\],?&])(?:["']?[a-z0-9_-]*(?:auth|cookie|credential|dsn|key|pass|private|secret|signature|token|url)[a-z0-9_-]*["']?)\s*(?:=|:)|--[a-z0-9-]*(?:auth|cookie|credential|dsn|key|pass|private|secret|signature|token|url)[a-z0-9-]*(?:=|\s+))/i; // --------------------------------------------------------------------------- // Helpers @@ -80,6 +91,14 @@ function tryUnlink(filePath: string): void { } } +function tryRemoveImportFailureDiagnostic(filePath: string): void { + try { + fs.unlinkSync(filePath); + } catch { + // absent diagnostics are expected after successful imports + } +} + function resolveKilocodeToken(): string | undefined { if (process.env.KILOCODE_TOKEN) { return process.env.KILOCODE_TOKEN; @@ -93,6 +112,104 @@ function resolveKilocodeToken(): string | undefined { return fs.readFileSync(tokenFile, 'utf8').replace(/[\r\n]+$/, ''); } +function stripUnsupportedControlCharacters(value: string): string { + return Array.from(value) + .filter(character => { + const code = character.charCodeAt(0); + return ( + character === '\t' || + character === '\n' || + character === '\r' || + (code >= 32 && code !== 127) + ); + }) + .join(''); +} + +function redactImportOutput( + output: string, + truncated: boolean, + additionalSensitiveValues: string[] +): string { + if (truncated) return OMITTED_TRUNCATED_OUTPUT; + + const environmentSecrets = Object.entries(process.env).flatMap(([name, value]) => + SENSITIVE_ENV_NAME.test(name) && typeof value === 'string' && value.length >= 4 ? [value] : [] + ); + let redacted = stripUnsupportedControlCharacters(output.replace(ANSI_ESCAPE_PATTERN, '')); + for (const value of [...environmentSecrets, ...additionalSensitiveValues] + .filter(value => value.length >= 4) + .sort((left, right) => right.length - left.length)) { + redacted = redacted.split(value).join(REDACTED); + } + + redacted = redacted.replace( + /-----BEGIN [^-\r\n]*PRIVATE KEY-----[\s\S]*?(?:-----END [^-\r\n]*PRIVATE KEY-----|$)/gi, + REDACTED + ); + redacted = redacted + .split('\n') + .map(line => (SENSITIVE_OUTPUT_LINE.test(line) ? REDACTED : line)) + .join('\n') + .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]*@/gi, `$1${REDACTED}@`) + .replace( + /([?&](?:access[_-]?token|api[_-]?key|key|secret|signature|token)=)[^&#\s]+/gi, + `$1${REDACTED}` + ); + return stripUnsupportedControlCharacters(redacted); +} + +function persistImportFailureDiagnostic( + diagnosticPath: string, + kiloSessionId: string, + result: ExecResult, + additionalSensitiveValues: string[] +): boolean { + const diagnostic = { + version: 1, + recordedAt: new Date().toISOString(), + kiloSessionId, + process: { + exitCode: result.exitCode, + ...(result.signal ? { signal: result.signal } : {}), + ...(result.terminationReason ? { terminationReason: result.terminationReason } : {}), + ...(result.elapsedMs === undefined ? {} : { elapsedMs: result.elapsedMs }), + }, + stdout: { + text: redactImportOutput( + result.stdout, + result.stdoutTruncated === true, + additionalSensitiveValues + ), + truncated: result.stdoutTruncated === true, + }, + stderr: { + text: redactImportOutput( + result.stderr, + result.stderrTruncated === true, + additionalSensitiveValues + ), + truncated: result.stderrTruncated === true, + }, + }; + + let temporaryDirectory: string | undefined; + try { + temporaryDirectory = fs.mkdtempSync(`${diagnosticPath}.tmp-`); + const temporaryPath = path.join(temporaryDirectory, 'diagnostic.json'); + fs.writeFileSync(temporaryPath, `${JSON.stringify(diagnostic, null, 2)}\n`, { + flag: 'wx', + mode: 0o600, + }); + fs.renameSync(temporaryPath, diagnosticPath); + return true; + } catch { + return false; + } finally { + if (temporaryDirectory) fs.rmSync(temporaryDirectory, { recursive: true, force: true }); + } +} + type SnapshotInfoValidation = 'valid' | 'missing' | 'invalid'; type SnapshotInfoValidationResult = { validation: SnapshotInfoValidation; @@ -498,6 +615,8 @@ export async function restoreSession( const tmpPath = filePath ?? `/tmp/kilo-session-export-${kiloSessionId}.json`; const downloaded = !filePath; const importTimeoutMs = options.importTimeoutMs ?? KILO_IMPORT_TIMEOUT_MS; + const diagnosticPath = options.diagnosticPath ?? getKiloImportDiagnosticPath(); + let kilocodeToken: string | undefined; log( `starting kiloSessionId=${kiloSessionId} workspace=${workspacePath} input=${downloaded ? 'downloaded' : 'provided'} tmpPath=${tmpPath} home=${process.env.HOME ?? '(unset)'}` @@ -505,15 +624,14 @@ export async function restoreSession( if (!filePath) { const ingestUrl = process.env.KILO_SESSION_INGEST_URL; - let token: string | undefined; try { - token = resolveKilocodeToken(); + kilocodeToken = resolveKilocodeToken(); } catch { return fail('failed to read KILOCODE_TOKEN_FILE', null, 'download'); } - if (!ingestUrl || !token) { - const missing = [!ingestUrl && 'KILO_SESSION_INGEST_URL', !token && 'KILOCODE_TOKEN'] + if (!ingestUrl || !kilocodeToken) { + const missing = [!ingestUrl && 'KILO_SESSION_INGEST_URL', !kilocodeToken && 'KILOCODE_TOKEN'] .filter(Boolean) .join(', '); return fail(`missing env vars: ${missing}`, null, 'download'); @@ -530,7 +648,7 @@ export async function restoreSession( ? AbortSignal.any([options.signal, downloadTimeoutSignal]) : downloadTimeoutSignal; const res = await fetch(url, { - headers: { Authorization: `Bearer ${token}` }, + headers: { Authorization: `Bearer ${kilocodeToken}` }, signal: downloadSignal, }); @@ -598,6 +716,18 @@ export async function restoreSession( }); const importElapsedMs = Date.now() - importStartedAt; + if (importResult.exitCode !== 0) { + const diagnosticPersisted = persistImportFailureDiagnostic( + diagnosticPath, + kiloSessionId, + importResult, + [...(options.sensitiveValues ?? []), ...(kilocodeToken ? [kilocodeToken] : [])] + ); + log( + `kilo import diagnostic ${diagnosticPersisted ? 'persisted' : 'persistence failed'} kiloSessionId=${kiloSessionId}` + ); + } + if (isTimeoutTermination(importResult)) { log( `kilo import finished outcome=timeout kiloSessionId=${kiloSessionId} input=${downloaded ? 'downloaded' : 'provided'} cwd=${workspacePath} home=${process.env.HOME ?? '(unset)'} elapsedMs=${importElapsedMs} timeoutMs=${importTimeoutMs}` @@ -626,6 +756,7 @@ export async function restoreSession( log( `kilo import finished outcome=ok exitCode=${importResult.exitCode} kiloSessionId=${kiloSessionId} input=${downloaded ? 'downloaded' : 'provided'} cwd=${workspacePath} home=${process.env.HOME ?? '(unset)'} elapsedMs=${importElapsedMs}` ); + tryRemoveImportFailureDiagnostic(diagnosticPath); // ---- Step 3: Apply diffs ---- // Extract diffs in a subprocess so the full snapshot JSON is never loaded diff --git a/services/cloud-agent-next/wrapper/src/session-bootstrap.test.ts b/services/cloud-agent-next/wrapper/src/session-bootstrap.test.ts index aca68cb9c5..cafde8aa26 100644 --- a/services/cloud-agent-next/wrapper/src/session-bootstrap.test.ts +++ b/services/cloud-agent-next/wrapper/src/session-bootstrap.test.ts @@ -13,6 +13,7 @@ import type { WrapperSessionReadyRequest, } from '../../src/shared/wrapper-bootstrap'; import { buildCloudAgentRules } from '../../src/shared/cloud-agent-rules.js'; +import { getKiloImportDiagnosticPath } from './utils.js'; function makeRequest(tmpDir: string, overrides: Partial = {}) { const request: WrapperSessionReadyRequest = { @@ -75,7 +76,9 @@ describe('prepareWrapperBootstrapWorkspace', () => { HOME: process.env.HOME, KILOCODE_TOKEN: process.env.KILOCODE_TOKEN, GH_TOKEN: process.env.GH_TOKEN, + WRAPPER_LOG_PATH: process.env.WRAPPER_LOG_PATH, }; + process.env.WRAPPER_LOG_PATH = path.join(tmpDir, 'trusted-wrapper.log'); }); afterEach(() => { @@ -155,6 +158,45 @@ describe('prepareWrapperBootstrapWorkspace', () => { ).toBe(true); }); + it('passes materialized environment values to import diagnostics', async () => { + const request = makeRequest(tmpDir); + request.materialized.setupCommands = []; + request.materialized.env.OPAQUE_NAME = JSON.stringify({ + token: 'nested-secret', + nested: { value: 'deep-secret' }, + }); + request.materialized.env.WRAPPER_LOG_PATH = path.join(tmpDir, 'untrusted-wrapper.log'); + let diagnosticPath: string | undefined; + let sensitiveValues: string[] | undefined; + + await prepareWrapperBootstrapWorkspace(request, undefined, { + git: async args => { + if (args[0] === 'clone') { + await fsp.mkdir(path.join(request.workspace.workspacePath, '.git'), { recursive: true }); + } + return { stdout: '', stderr: '', exitCode: args[0] === 'rev-parse' ? 1 : 0 }; + }, + restoreSession: async (_kiloSessionId, _workspacePath, _filePath, options) => { + diagnosticPath = options?.diagnosticPath; + sensitiveValues = options?.sensitiveValues; + return { + ok: true, + downloaded: false, + imported: true, + diffs: { applied: 0, skipped: 0, total: 0 }, + }; + }, + }); + + expect(diagnosticPath).toBe( + getKiloImportDiagnosticPath(path.join(tmpDir, 'trusted-wrapper.log')) + ); + expect(process.env.WRAPPER_LOG_PATH).toBe(path.join(tmpDir, 'trusted-wrapper.log')); + expect(new Set(sensitiveValues)).toEqual( + new Set([...Object.values(request.materialized.env), 'nested-secret', 'deep-secret']) + ); + }); + it('uses activity watchdogs and reports sanitized progress for long git operations', async () => { const request = makeRequest(tmpDir); request.materialized.setupCommands = []; diff --git a/services/cloud-agent-next/wrapper/src/session-bootstrap.ts b/services/cloud-agent-next/wrapper/src/session-bootstrap.ts index e2eef81540..88795595a5 100644 --- a/services/cloud-agent-next/wrapper/src/session-bootstrap.ts +++ b/services/cloud-agent-next/wrapper/src/session-bootstrap.ts @@ -8,7 +8,9 @@ import { } from '../../src/shared/wrapper-bootstrap.js'; import { buildCloudAgentRules } from '../../src/shared/cloud-agent-rules.js'; import { + applyMaterializedEnvironment, createSafeProcessDiagnostic, + getKiloImportDiagnosticPath, git, isTimeoutTermination, logToFile, @@ -443,9 +445,38 @@ async function writeRuntimeSkills(request: WrapperSessionReadyRequest): Promise< } } +function collectNestedStringValues(value: unknown, values: Set): void { + if (typeof value === 'string') { + if (value.length >= 4) values.add(value); + return; + } + if (Array.isArray(value)) { + for (const entry of value) collectNestedStringValues(entry, values); + return; + } + if (typeof value === 'object' && value !== null) { + for (const entry of Object.values(value)) collectNestedStringValues(entry, values); + } +} + +function collectMaterializedSensitiveValues(env: Record): string[] { + const values = new Set(); + for (const serialized of Object.values(env)) { + if (serialized.length >= 4) values.add(serialized); + try { + const parsed: unknown = JSON.parse(serialized); + collectNestedStringValues(parsed, values); + } catch { + // Non-JSON environment values are already included verbatim. + } + } + return [...values]; +} + async function bootstrapEmptyKiloSession( request: WrapperSessionReadyRequest, - restore: typeof restoreSession + restore: typeof restoreSession, + diagnosticPath: string ): Promise { const now = Date.now(); const minimalSessionJson = JSON.stringify({ @@ -468,7 +499,11 @@ async function bootstrapEmptyKiloSession( const result = await restore( request.kiloSessionId, request.workspace.workspacePath, - importFilePath + importFilePath, + { + diagnosticPath, + sensitiveValues: collectMaterializedSensitiveValues(request.materialized.env), + } ); if (!result.ok) { logToFile( @@ -489,13 +524,22 @@ async function bootstrapEmptyKiloSession( async function restoreOrBootstrapKiloSession( request: WrapperSessionReadyRequest, - restore: typeof restoreSession + restore: typeof restoreSession, + diagnosticPath: string ): Promise { if (request.workspace.preferSnapshot) { logToFile( `bootstrap snapshot restore starting kiloSessionId=${request.kiloSessionId} workspacePath=${request.workspace.workspacePath}` ); - const result = await restore(request.kiloSessionId, request.workspace.workspacePath); + const result = await restore( + request.kiloSessionId, + request.workspace.workspacePath, + undefined, + { + diagnosticPath, + sensitiveValues: collectMaterializedSensitiveValues(request.materialized.env), + } + ); if (result.ok) { logToFile( `bootstrap snapshot restore ready kiloSessionId=${request.kiloSessionId} downloaded=${result.downloaded} diffsApplied=${result.diffs.applied} diffsSkipped=${result.diffs.skipped} diffsTotal=${result.diffs.total}` @@ -520,7 +564,7 @@ async function restoreOrBootstrapKiloSession( } else { logToFile(`bootstrap fresh session using empty import kiloSessionId=${request.kiloSessionId}`); } - await bootstrapEmptyKiloSession(request, restore); + await bootstrapEmptyKiloSession(request, restore, diagnosticPath); } async function runSetupCommands( @@ -630,8 +674,9 @@ async function prepareWrapperBootstrapWorkspaceWithinDeadline( const runGit = deps.git ?? git; const run = deps.runProcess ?? runProcess; const restore = deps.restoreSession ?? restoreSession; + const diagnosticPath = getKiloImportDiagnosticPath(); - Object.assign(process.env, request.materialized.env); + applyMaterializedEnvironment(request.materialized.env); let workspaceWasWarm = false; let workspaceNeedsBootstrap = true; @@ -687,7 +732,7 @@ async function prepareWrapperBootstrapWorkspaceWithinDeadline( 'kilo_session', request.workspace.preferSnapshot ? 'Restoring session...' : 'Importing session...' ); - await restoreOrBootstrapKiloSession(request, restore); + await restoreOrBootstrapKiloSession(request, restore, diagnosticPath); if (request.materialized.setupCommands?.length) { progress?.('setup_commands', 'Running setup commands...'); diff --git a/services/cloud-agent-next/wrapper/src/utils.ts b/services/cloud-agent-next/wrapper/src/utils.ts index 1e633bd136..7b2ec2405d 100644 --- a/services/cloud-agent-next/wrapper/src/utils.ts +++ b/services/cloud-agent-next/wrapper/src/utils.ts @@ -5,6 +5,7 @@ export type ExecResult = { stdout: string; stderr: string; exitCode: number; + signal?: NodeJS.Signals; elapsedMs?: number; terminationReason?: TerminationReason; stdoutTruncated?: boolean; @@ -41,10 +42,26 @@ const EXEC_INACTIVITY_TIMEOUT_MESSAGE = 'exec inactivity timeout reached'; const EXEC_HARD_TIMEOUT_MESSAGE = 'exec hard timeout reached'; const EXEC_ABORTED_MESSAGE = 'exec aborted'; const DEFAULT_MAX_OUTPUT_BYTES = 64 * 1_024; +const DEFAULT_WRAPPER_LOG_PATH = '/tmp/kilocode-wrapper.log'; +const KILO_IMPORT_DIAGNOSTIC_SUFFIX = '.kilo-import-failure.json'; const TRUNCATION_MARKER = 'output truncated'; export type TerminationReason = 'timeout' | 'inactivity_timeout' | 'hard_timeout' | 'abort'; +export function getKiloImportDiagnosticPath(wrapperLogPath?: string): string { + return `${wrapperLogPath ?? process.env.WRAPPER_LOG_PATH ?? DEFAULT_WRAPPER_LOG_PATH}${KILO_IMPORT_DIAGNOSTIC_SUFFIX}`; +} + +export function applyMaterializedEnvironment(env: Record): void { + const wrapperLogPath = process.env.WRAPPER_LOG_PATH; + Object.assign(process.env, env); + if (wrapperLogPath === undefined) { + delete process.env.WRAPPER_LOG_PATH; + } else { + process.env.WRAPPER_LOG_PATH = wrapperLogPath; + } +} + export function isTimeoutTermination(result: ExecResult): boolean { return ( result.terminationReason === 'timeout' || @@ -80,6 +97,7 @@ export function createSafeProcessDiagnostic(result: ExecResult): string { result.terminationReason === undefined && result.exitCode !== 0 ? `exit code ${result.exitCode}` : undefined, + result.signal === undefined ? undefined : `signal ${result.signal}`, result.elapsedMs === undefined ? undefined : `elapsed ${result.elapsedMs}ms`, result.stdoutTruncated === true || result.stderrTruncated === true ? TRUNCATION_MARKER @@ -277,6 +295,7 @@ export function runProcess( stdout, stderr, exitCode: code ?? (signal === null ? 0 : 1), + ...(signal === null ? {} : { signal }), elapsedMs: Date.now() - startedAt, ...(stdoutTruncated ? { stdoutTruncated: true } : {}), ...(stderrTruncated ? { stderrTruncated: true } : {}),