diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 2ba94481..a97868fb 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1499,13 +1499,16 @@ export async function main( }, createSecurity: dependencies.createSecurity, signal: controller.signal, - onProgress: ({ repository, status, attempt, error }) => { + onProgress: ({ repository, status, attempt, error, warning }) => { + const detail = error ?? warning; errorOutput.write( - `codex-security: ${repository} ${status} (attempt ${attempt})${error === undefined ? "" : `: ${redactedErrorMessage(error)}`}\n`, + `codex-security: ${repository} ${status} (attempt ${attempt})${detail === undefined ? "" : `: ${redactedErrorMessage(detail)}`}\n`, ); }, }); - exitCode = interruptedExitCode() ?? (result.failed > 0 ? 2 : 0); + exitCode = + interruptedExitCode() ?? + (result.failed > 0 || result.incomplete > 0 ? 2 : 0); return { ...result }; } catch (error) { exitCode = diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts index c0ae08af..9704014c 100644 --- a/sdk/typescript/src/multiscan.ts +++ b/sdk/typescript/src/multiscan.ts @@ -20,6 +20,7 @@ import type { CodexSecurity } from "./api.js"; import type { CodexSecurityConfig } from "./config.js"; import type { ScanCost } from "./cost.js"; import { redactedErrorMessage } from "./errors.js"; +import type { CoverageDocument } from "./models.js"; import type { ScanMode } from "./targets.js"; import { resolveTrustedExecutable } from "./trusted-executable.js"; @@ -43,11 +44,13 @@ interface MultiscanTask { } interface MultiscanReceipt extends MultiscanTask { - status: "completed" | "failed"; + status: "completed" | "completed_with_incomplete_coverage" | "failed"; attempt: number; outputDir: string; + coverage?: CoverageDocument["completeness"]; cost?: ScanCost; error?: string; + warning?: string; } export interface MultiscanOptions { @@ -67,15 +70,21 @@ export interface MultiscanOptions { signal?: AbortSignal; onProgress?(event: { repository: string; - status: "started" | "completed" | "failed"; + status: + | "started" + | "completed" + | "completed_with_incomplete_coverage" + | "failed"; attempt: number; error?: string; + warning?: string; }): void; } export interface MultiscanResult { total: number; completed: number; + incomplete: number; failed: number; skipped: number; resultsPath: string; @@ -118,24 +127,44 @@ async function runCampaign( const receipts = await readReceipts(ledger); const pending: MultiscanTask[] = []; let completed = 0; + let incomplete = 0; for (const task of tasks) { const receipt = receipts.get(task.id.toLowerCase()); if ( - receipt?.status === "completed" && + receipt !== undefined && receipt.outputDir === join(output, "artifacts", task.id, `attempt-${receipt.attempt}`) && (await hasArtifacts(receipt.outputDir)) ) { - completed += 1; - } else { - pending.push(task); + if (receipt.status === "completed") { + completed += 1; + continue; + } + const coverage = + receipt.status === "completed_with_incomplete_coverage" + ? receipt.coverage ?? "unknown" + : await legacyIncompleteCoverage(receipt); + if (coverage !== undefined) { + incomplete += 1; + options.onProgress?.({ + repository: task.id, + status: "completed_with_incomplete_coverage", + attempt: receipt.attempt, + warning: + receipt.warning ?? + `Scan coverage is ${coverage}; results may be incomplete.`, + }); + continue; + } } + pending.push(task); } - const skipped = completed; + const skipped = completed + incomplete; if (pending.length === 0) { return { total: tasks.length, completed, + incomplete, failed: 0, skipped, resultsPath: ledger, @@ -165,6 +194,8 @@ async function runCampaign( const progress = { repository: task.id, attempt }; options.onProgress?.({ ...progress, status: "started" }); let failure: string | undefined; + let warning: string | undefined; + let coverage: CoverageDocument["completeness"] | undefined; let cost: Readonly | null = null; try { await mkdir(dirname(scanDir), { recursive: true, mode: 0o700 }); @@ -204,8 +235,14 @@ async function runCampaign( ...(options.signal === undefined ? {} : { signal: options.signal }), }); cost = result.cost; - if (result.coverage.completeness !== "complete") { - throw new Error("Multiscan repository coverage is incomplete."); + coverage = result.coverage.completeness; + if (coverage !== "complete") { + if (!(await hasArtifacts(scanDir))) { + throw new Error( + "Multiscan scan output is missing required artifacts.", + ); + } + warning = `Scan coverage is ${coverage}; results may be incomplete.`; } } catch (error) { if (options.signal?.aborted === true) options.signal.throwIfAborted(); @@ -213,7 +250,12 @@ async function runCampaign( } finally { await rm(checkout, { recursive: true, force: true }); } - const status = failure === undefined ? "completed" : "failed"; + const status = + failure !== undefined + ? "failed" + : warning === undefined + ? "completed" + : "completed_with_incomplete_coverage"; await appendReceipt( ledger, `${JSON.stringify({ @@ -221,17 +263,21 @@ async function runCampaign( status, attempt, outputDir: scanDir, + ...(coverage === undefined ? {} : { coverage }), ...(cost === null ? {} : { cost }), ...(failure === undefined ? {} : { error: failure }), + ...(warning === undefined ? {} : { warning }), })}\n`, ); options.onProgress?.({ ...progress, status, ...(failure === undefined ? {} : { error: failure }), + ...(warning === undefined ? {} : { warning }), }); if (failure === undefined) { - completed += 1; + if (warning === undefined) completed += 1; + else incomplete += 1; break; } if (retry === options.maxAttempts - 1) failed += 1; @@ -256,6 +302,7 @@ async function runCampaign( return { total: tasks.length, completed, + incomplete, failed, skipped, resultsPath: ledger, @@ -508,6 +555,28 @@ async function hasArtifacts(path: string): Promise { } } +async function legacyIncompleteCoverage( + receipt: MultiscanReceipt, +): Promise | undefined> { + if ( + receipt.status !== "failed" || + receipt.error !== "Multiscan repository coverage is incomplete." + ) { + return undefined; + } + try { + const coverage = JSON.parse( + await readFile(join(receipt.outputDir, "coverage.json"), "utf8"), + ) as { completeness?: unknown }; + return coverage.completeness === "partial" || + coverage.completeness === "unknown" + ? coverage.completeness + : undefined; + } catch { + return undefined; + } +} + function parseInventory( source: string, directory: string, diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index 21cb3cb6..3ef41a0b 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -17,9 +17,11 @@ import * as filesystem from "node:fs/promises"; import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { main } from "../src/cli.js"; import type { ScanResult } from "../src/result.js"; import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js"; import { resolveTrustedExecutable } from "../src/trusted-executable.js"; +import { capture, dependencies, fakeResult } from "./cli-fixtures.js"; type MultiscanOptions = Parameters[0]; type SecurityClient = ReturnType; @@ -82,7 +84,7 @@ async function repository( async function completedScan( outputDir: string, - completeness: "complete" | "partial" = "complete", + completeness: "complete" | "partial" | "unknown" = "complete", ): Promise { await mkdir(outputDir, { recursive: true }); await Promise.all( @@ -251,8 +253,383 @@ describe("multiscan", () => { ), ); + expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 }); expect(await results(summary.resultsPath)).toMatchObject([ - { id: "priced", status: "completed", cost }, + { id: "priced", status: "completed", coverage: "complete", cost }, + ]); + }); + + test.each(["partial", "unknown"] as const)( + "retains sealed %s coverage without retries or multiplied costs", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, completeness); + await writeFile( + paths.input, + `id,repository,revision\nsealed,${source.path},${source.revision}\n`, + ); + const cost = { + model: "gpt-5.6-sol", + inputTokens: 1_250, + cachedInputTokens: 200, + cacheWriteInputTokens: 0, + outputTokens: 30, + estimatedUsd: 12.5, + }; + const progress: Parameters< + NonNullable + >[0][] = []; + let attempts = 0; + const security = client(async (_repository, scanOptions = {}) => { + attempts += 1; + return Object.assign( + await completedScan(scanOptions.outputDir!, completeness), + { cost }, + ); + }); + + const summary = await runMultiscan( + options(paths, security, { + maxAttempts: 3, + onProgress: (event) => progress.push(event), + }), + ); + + expect(attempts).toBe(1); + expect(summary).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 0, + }); + const outputDir = join(paths.output, "artifacts", "sealed", "attempt-1"); + const warning = `Scan coverage is ${completeness}; results may be incomplete.`; + const receipts = await results(summary.resultsPath); + expect(receipts).toMatchObject([ + { + id: "sealed", + status: "completed_with_incomplete_coverage", + attempt: 1, + outputDir, + coverage: completeness, + cost, + warning, + }, + ]); + expect( + receipts.reduce( + (total, receipt) => + total + (receipt["cost"] as typeof cost).estimatedUsd, + 0, + ), + ).toBe(cost.estimatedUsd); + await Promise.all( + [ + "scan-manifest.json", + "findings.json", + "coverage.json", + "report.md", + ].map((name) => access(join(outputDir, name))), + ); + expect(progress).toMatchObject([ + { repository: "sealed", status: "started", attempt: 1 }, + { + repository: "sealed", + status: "completed_with_incomplete_coverage", + attempt: 1, + warning, + }, + ]); + + const resumed = await runMultiscan( + options(paths, security, { maxAttempts: 3 }), + ); + expect(resumed).toMatchObject({ + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(attempts).toBe(1); + expect(await results(resumed.resultsPath)).toHaveLength(1); + }, + ); + + test.each(["partial", "unknown"] as const)( + "resumes legacy sealed %s coverage without rerunning or duplicating cost", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, `legacy-${completeness}`); + await writeFile( + paths.input, + `id,repository,revision\nlegacy,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1"); + await completedScan(outputDir, completeness); + await writeFile( + join(outputDir, "coverage.json"), + `${JSON.stringify({ completeness })}\n`, + ); + const cost = { + model: "gpt-5.6-sol", + inputTokens: 1_250, + cachedInputTokens: 200, + cacheWriteInputTokens: 0, + outputTokens: 30, + estimatedUsd: 231.73, + }; + const receipt = { + id: "legacy", + repository: source.path, + revision: source.revision, + mode: "standard", + status: "failed", + attempt: 1, + outputDir, + cost, + error: "Multiscan repository coverage is incomplete.", + }; + await writeFile( + join(paths.output, "results.jsonl"), + `${JSON.stringify(receipt)}\n`, + ); + const progress: Parameters< + NonNullable + >[0][] = []; + let attempts = 0; + const security = client(async (_repository, scanOptions = {}) => { + attempts += 1; + return await completedScan(scanOptions.outputDir!); + }); + + const summary = await runMultiscan( + options(paths, security, { + maxAttempts: 3, + onProgress: (event) => progress.push(event), + }), + ); + + expect(summary).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(attempts).toBe(0); + expect(progress).toEqual([ + { + repository: "legacy", + status: "completed_with_incomplete_coverage", + attempt: 1, + warning: `Scan coverage is ${completeness}; results may be incomplete.`, + }, + ]); + expect(await results(summary.resultsPath)).toEqual([receipt]); + + await runMultiscan(options(paths, security, { maxAttempts: 3 })); + expect(attempts).toBe(0); + expect(await results(summary.resultsPath)).toEqual([receipt]); + }, + ); + + test.each([ + ["operational failures", "partial", "Worker exited unexpectedly.", false], + [ + "complete coverage", + "complete", + "Multiscan repository coverage is incomplete.", + false, + ], + [ + "malformed coverage", + "malformed", + "Multiscan repository coverage is incomplete.", + false, + ], + [ + "missing artifacts", + "partial", + "Multiscan repository coverage is incomplete.", + true, + ], + ] as const)( + "continues retrying legacy %s", + async (_scenario, completeness, error, missingArtifact) => { + const paths = await fixture(); + const source = await repository(paths.root, "legacy-retry"); + await writeFile( + paths.input, + `id,repository,revision\nlegacy,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1"); + await completedScan(outputDir); + await writeFile( + join(outputDir, "coverage.json"), + completeness === "malformed" + ? "{\n" + : `${JSON.stringify({ completeness })}\n`, + ); + if (missingArtifact) await rm(join(outputDir, "report.md")); + await writeFile( + join(paths.output, "results.jsonl"), + `${JSON.stringify({ + id: "legacy", + repository: source.path, + revision: source.revision, + mode: "standard", + status: "failed", + attempt: 1, + outputDir, + error, + })}\n`, + ); + let attempts = 0; + + const summary = await runMultiscan( + options( + paths, + client(async (_repository, scanOptions = {}) => { + attempts += 1; + return await completedScan(scanOptions.outputDir!); + }), + ), + ); + + expect(summary).toMatchObject({ + completed: 1, + incomplete: 0, + failed: 0, + skipped: 0, + }); + expect(attempts).toBe(1); + expect(await results(summary.resultsPath)).toMatchObject([ + { status: "failed", attempt: 1, error }, + { status: "completed", attempt: 2, coverage: "complete" }, + ]); + }, + ); + + test.each(["partial", "unknown"] as const)( + "keeps sealed %s-coverage CLI runs fail-closed without retrying", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, "sample"); + await writeFile( + paths.input, + `id,repository,revision\nsample,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "sample", "attempt-1"); + await completedScan(outputDir, completeness); + const stdout = capture(); + const stderr = capture(); + let attempts = 0; + const arguments_ = [ + "bulk-scan", + "repositories.csv", + "--output-dir", + "results", + "--max-attempts", + "3", + "--json", + ]; + const clientDependencies = dependencies({ + currentDirectory: paths.root, + result: fakeResult([], completeness), + onRun: () => { + attempts += 1; + }, + }); + + expect( + await main( + arguments_, + stdout.stream, + stderr.stream, + clientDependencies, + ), + ).toBe(2); + expect(attempts).toBe(1); + expect(JSON.parse(stdout.text())).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 0, + }); + const warning = `Scan coverage is ${completeness}; results may be incomplete.`; + expect(stderr.text()).toContain( + "sample completed_with_incomplete_coverage (attempt 1)", + ); + expect(stderr.text()).toContain(warning); + expect(stderr.text()).not.toContain("attempt 2"); + expect(await results(join(paths.output, "results.jsonl"))).toMatchObject([ + { + status: "completed_with_incomplete_coverage", + coverage: completeness, + outputDir, + }, + ]); + + const resumedOutput = capture(); + const resumedError = capture(); + expect( + await main( + arguments_, + resumedOutput.stream, + resumedError.stream, + clientDependencies, + ), + ).toBe(2); + expect(JSON.parse(resumedOutput.text())).toMatchObject({ + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(resumedError.text()).toContain(warning); + expect(attempts).toBe(1); + }, + ); + + test("retries incomplete scans that are missing required artifacts", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "missing-artifact"); + await writeFile( + paths.input, + `id,repository,revision\nmissing,${source.path},${source.revision}\n`, + ); + + let attempts = 0; + const summary = await runMultiscan( + options( + paths, + client(async (_repository, scanOptions = {}) => { + attempts += 1; + const result = await completedScan( + scanOptions.outputDir!, + attempts === 1 ? "partial" : "complete", + ); + if (attempts === 1) { + await rm(join(scanOptions.outputDir!, "report.md")); + } + return result; + }), + ), + ); + + expect(attempts).toBe(2); + expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 }); + expect(await results(summary.resultsPath)).toMatchObject([ + { + id: "missing", + status: "failed", + attempt: 1, + coverage: "partial", + error: "Multiscan scan output is missing required artifacts.", + }, + { id: "missing", status: "completed", attempt: 2, coverage: "complete" }, ]); }); @@ -1109,7 +1486,7 @@ describe("multiscan", () => { expect(scans).toBe(0); }); - test("treats incomplete coverage as a failure and still finishes other repositories", async () => { + test("records incomplete coverage separately and still finishes other repositories", async () => { const paths = await fixture(); const incomplete = await repository(paths.root, "incomplete"); const complete = await repository(paths.root, "complete"); @@ -1136,14 +1513,24 @@ describe("multiscan", () => { : "complete", ), ), - { maxAttempts: 1 }, + { maxAttempts: 3 }, ), ); - expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 }); + expect(summary).toMatchObject({ + total: 2, + completed: 1, + incomplete: 1, + failed: 0, + }); expect(await results(summary.resultsPath)).toMatchObject([ - { id: "incomplete", status: "failed", attempt: 1 }, - { id: "complete", status: "completed", attempt: 1 }, + { + id: "incomplete", + status: "completed_with_incomplete_coverage", + attempt: 1, + coverage: "partial", + }, + { id: "complete", status: "completed", attempt: 1, coverage: "complete" }, ]); }); });