From 65724b0e976b3f2633163de15be9cbe8b1511985 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Wed, 5 Aug 2026 21:52:54 +0000 Subject: [PATCH 1/3] fix: stop retrying sealed incomplete bulk scans --- sdk/typescript/src/cli.ts | 9 +- sdk/typescript/src/multiscan.ts | 60 ++++++-- sdk/typescript/tests-ts/cli.test.ts | 104 ++++++++++++++ sdk/typescript/tests-ts/multiscan.test.ts | 162 +++++++++++++++++++++- 4 files changed, 316 insertions(+), 19 deletions(-) diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index bf905ad9..fa3849a0 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1451,13 +1451,16 @@ export async function main( }, createSecurity: dependencies.createSecurity, signal: controller.signal, - onProgress: ({ repository, status, attempt, error }) => { + onProgress: ({ repository, status, attempt, error, warning }) => { + const detail = error ?? warning; errorOutput.write( - `codex-security: ${repository} ${status} (attempt ${attempt})${error === undefined ? "" : `: ${redactedErrorMessage(error)}`}\n`, + `codex-security: ${repository} ${status} (attempt ${attempt})${detail === undefined ? "" : `: ${redactedErrorMessage(detail)}`}\n`, ); }, }); - exitCode = interruptedExitCode() ?? (result.failed > 0 ? 2 : 0); + exitCode = + interruptedExitCode() ?? + (result.failed > 0 || result.incomplete > 0 ? 2 : 0); return { ...result }; } catch (error) { exitCode = diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts index 4795bcfd..de6e8b49 100644 --- a/sdk/typescript/src/multiscan.ts +++ b/sdk/typescript/src/multiscan.ts @@ -18,6 +18,7 @@ import type { CodexSecurity } from "./api.js"; import type { CodexSecurityConfig } from "./config.js"; import type { ScanCost } from "./cost.js"; import { redactedErrorMessage } from "./errors.js"; +import type { CoverageDocument } from "./models.js"; import type { ScanMode } from "./targets.js"; import { resolveTrustedExecutable } from "./trusted-executable.js"; @@ -38,11 +39,13 @@ interface MultiscanTask { } interface MultiscanReceipt extends MultiscanTask { - status: "completed" | "failed"; + status: "completed" | "completed_with_incomplete_coverage" | "failed"; attempt: number; outputDir: string; + coverage?: CoverageDocument["completeness"]; cost?: ScanCost; error?: string; + warning?: string; } export interface MultiscanOptions { @@ -60,15 +63,21 @@ export interface MultiscanOptions { signal?: AbortSignal; onProgress?(event: { repository: string; - status: "started" | "completed" | "failed"; + status: + | "started" + | "completed" + | "completed_with_incomplete_coverage" + | "failed"; attempt: number; error?: string; + warning?: string; }): void; } export interface MultiscanResult { total: number; completed: number; + incomplete: number; failed: number; skipped: number; resultsPath: string; @@ -111,24 +120,39 @@ async function runCampaign( const receipts = await readReceipts(ledger); const pending: MultiscanTask[] = []; let completed = 0; + let incomplete = 0; for (const task of tasks) { const receipt = receipts.get(task.id.toLowerCase()); if ( - receipt?.status === "completed" && + (receipt?.status === "completed" || + receipt?.status === "completed_with_incomplete_coverage") && receipt.outputDir === join(output, "artifacts", task.id, `attempt-${receipt.attempt}`) && (await hasArtifacts(receipt.outputDir)) ) { - completed += 1; + if (receipt.status === "completed") { + completed += 1; + } else { + incomplete += 1; + options.onProgress?.({ + repository: task.id, + status: receipt.status, + attempt: receipt.attempt, + warning: + receipt.warning ?? + `Scan coverage is ${receipt.coverage ?? "unknown"}; results may be incomplete.`, + }); + } } else { pending.push(task); } } - const skipped = completed; + const skipped = completed + incomplete; if (pending.length === 0) { return { total: tasks.length, completed, + incomplete, failed: 0, skipped, resultsPath: ledger, @@ -158,6 +182,8 @@ async function runCampaign( const progress = { repository: task.id, attempt }; options.onProgress?.({ ...progress, status: "started" }); let failure: string | undefined; + let warning: string | undefined; + let coverage: CoverageDocument["completeness"] | undefined; let cost: Readonly | null = null; try { await mkdir(dirname(scanDir), { recursive: true, mode: 0o700 }); @@ -190,8 +216,14 @@ async function runCampaign( ...(options.signal === undefined ? {} : { signal: options.signal }), }); cost = result.cost; - if (result.coverage.completeness !== "complete") { - throw new Error("Multiscan repository coverage is incomplete."); + coverage = result.coverage.completeness; + if (coverage !== "complete") { + if (!(await hasArtifacts(scanDir))) { + throw new Error( + "Multiscan scan output is missing required artifacts.", + ); + } + warning = `Scan coverage is ${coverage}; results may be incomplete.`; } } catch (error) { if (options.signal?.aborted === true) options.signal.throwIfAborted(); @@ -199,7 +231,12 @@ async function runCampaign( } finally { await rm(checkout, { recursive: true, force: true }); } - const status = failure === undefined ? "completed" : "failed"; + const status = + failure !== undefined + ? "failed" + : warning === undefined + ? "completed" + : "completed_with_incomplete_coverage"; await appendReceipt( ledger, `${JSON.stringify({ @@ -207,17 +244,21 @@ async function runCampaign( status, attempt, outputDir: scanDir, + ...(coverage === undefined ? {} : { coverage }), ...(cost === null ? {} : { cost }), ...(failure === undefined ? {} : { error: failure }), + ...(warning === undefined ? {} : { warning }), })}\n`, ); options.onProgress?.({ ...progress, status, ...(failure === undefined ? {} : { error: failure }), + ...(warning === undefined ? {} : { warning }), }); if (failure === undefined) { - completed += 1; + if (warning === undefined) completed += 1; + else incomplete += 1; break; } if (retry === options.maxAttempts - 1) failed += 1; @@ -242,6 +283,7 @@ async function runCampaign( return { total: tasks.length, completed, + incomplete, failed, skipped, resultsPath: ledger, diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index 725e8b42..ec8288ba 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -759,6 +759,110 @@ describe("CLI", () => { } }); + test.each(["partial", "unknown"] as const)( + "keeps sealed %s-coverage bulk scans fail-closed without retrying", + async (completeness) => { + const root = await mkdtemp( + join(tmpdir(), "codex-security-cli-multiscan-incomplete-"), + ); + try { + await multiscanInventory(root); + const outputDir = join( + root, + "results", + "artifacts", + "sample", + "attempt-1", + ); + await mkdir(outputDir, { recursive: true }); + await Promise.all( + [ + "scan-manifest.json", + "findings.json", + "coverage.json", + "report.md", + ].map((name) => writeFile(join(outputDir, name), "{}\n")), + ); + const stdout = capture(); + const stderr = capture(); + let attempts = 0; + const arguments_ = [ + "bulk-scan", + "repositories.csv", + "--output-dir", + "results", + "--max-attempts", + "3", + "--json", + ]; + const clientDependencies = dependencies({ + currentDirectory: root, + result: fakeResult([], completeness), + onRun: () => { + attempts += 1; + }, + }); + + expect( + await main( + arguments_, + stdout.stream, + stderr.stream, + clientDependencies, + ), + ).toBe(2); + expect(attempts).toBe(1); + expect(JSON.parse(stdout.text())).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 0, + }); + expect(stderr.text()).toContain( + "sample completed_with_incomplete_coverage (attempt 1)", + ); + expect(stderr.text()).toContain( + `Scan coverage is ${completeness}; results may be incomplete.`, + ); + expect(stderr.text()).not.toContain("attempt 2"); + const receipt = JSON.parse( + ( + await readFile(join(root, "results", "results.jsonl"), "utf8") + ).trim(), + ) as Record; + expect(receipt).toMatchObject({ + status: "completed_with_incomplete_coverage", + coverage: completeness, + outputDir, + }); + + const resumedOutput = capture(); + const resumedError = capture(); + expect( + await main( + arguments_, + resumedOutput.stream, + resumedError.stream, + clientDependencies, + ), + ).toBe(2); + expect(JSON.parse(resumedOutput.text())).toMatchObject({ + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(resumedError.text()).toContain( + `Scan coverage is ${completeness}; results may be incomplete.`, + ); + expect(attempts).toBe(1); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + ); + test.each([ [ "OpenRouter", diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index 429bb793..fce082e7 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -78,7 +78,7 @@ async function repository( async function completedScan( outputDir: string, - completeness: "complete" | "partial" = "complete", + completeness: "complete" | "partial" | "unknown" = "complete", ): Promise { await mkdir(outputDir, { recursive: true }); await Promise.all( @@ -230,8 +230,146 @@ describe("multiscan", () => { ), ); + expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 }); expect(await results(summary.resultsPath)).toMatchObject([ - { id: "priced", status: "completed", cost }, + { id: "priced", status: "completed", coverage: "complete", cost }, + ]); + }); + + test.each(["partial", "unknown"] as const)( + "retains sealed %s coverage without retries or multiplied costs", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, completeness); + await writeFile( + paths.input, + `id,repository,revision\nsealed,${source.path},${source.revision}\n`, + ); + const cost = { + model: "gpt-5.6-sol", + inputTokens: 1_250, + cachedInputTokens: 200, + cacheWriteInputTokens: 0, + outputTokens: 30, + estimatedUsd: 12.5, + }; + const progress: Parameters< + NonNullable + >[0][] = []; + let attempts = 0; + const security = client(async (_repository, scanOptions = {}) => { + attempts += 1; + return Object.assign( + await completedScan(scanOptions.outputDir!, completeness), + { cost }, + ); + }); + + const summary = await runMultiscan( + options(paths, security, { + maxAttempts: 3, + onProgress: (event) => progress.push(event), + }), + ); + + expect(attempts).toBe(1); + expect(summary).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 0, + }); + const outputDir = join(paths.output, "artifacts", "sealed", "attempt-1"); + const warning = `Scan coverage is ${completeness}; results may be incomplete.`; + const receipts = await results(summary.resultsPath); + expect(receipts).toMatchObject([ + { + id: "sealed", + status: "completed_with_incomplete_coverage", + attempt: 1, + outputDir, + coverage: completeness, + cost, + warning, + }, + ]); + expect( + receipts.reduce( + (total, receipt) => + total + (receipt["cost"] as typeof cost).estimatedUsd, + 0, + ), + ).toBe(cost.estimatedUsd); + await Promise.all( + [ + "scan-manifest.json", + "findings.json", + "coverage.json", + "report.md", + ].map((name) => access(join(outputDir, name))), + ); + expect(progress).toMatchObject([ + { repository: "sealed", status: "started", attempt: 1 }, + { + repository: "sealed", + status: "completed_with_incomplete_coverage", + attempt: 1, + warning, + }, + ]); + + const resumed = await runMultiscan( + options(paths, security, { maxAttempts: 3 }), + ); + expect(resumed).toMatchObject({ + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(attempts).toBe(1); + expect(await results(resumed.resultsPath)).toHaveLength(1); + }, + ); + + test("retries incomplete scans that are missing required artifacts", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "missing-artifact"); + await writeFile( + paths.input, + `id,repository,revision\nmissing,${source.path},${source.revision}\n`, + ); + + let attempts = 0; + const summary = await runMultiscan( + options( + paths, + client(async (_repository, scanOptions = {}) => { + attempts += 1; + const result = await completedScan( + scanOptions.outputDir!, + attempts === 1 ? "partial" : "complete", + ); + if (attempts === 1) { + await rm(join(scanOptions.outputDir!, "report.md")); + } + return result; + }), + ), + ); + + expect(attempts).toBe(2); + expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 }); + expect(await results(summary.resultsPath)).toMatchObject([ + { + id: "missing", + status: "failed", + attempt: 1, + coverage: "partial", + error: "Multiscan scan output is missing required artifacts.", + }, + { id: "missing", status: "completed", attempt: 2, coverage: "complete" }, ]); }); @@ -705,7 +843,7 @@ describe("multiscan", () => { expect(scans).toBe(0); }); - test("treats incomplete coverage as a failure and still finishes other repositories", async () => { + test("records incomplete coverage separately and still finishes other repositories", async () => { const paths = await fixture(); const incomplete = await repository(paths.root, "incomplete"); const complete = await repository(paths.root, "complete"); @@ -732,14 +870,24 @@ describe("multiscan", () => { : "complete", ), ), - { maxAttempts: 1 }, + { maxAttempts: 3 }, ), ); - expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 }); + expect(summary).toMatchObject({ + total: 2, + completed: 1, + incomplete: 1, + failed: 0, + }); expect(await results(summary.resultsPath)).toMatchObject([ - { id: "incomplete", status: "failed", attempt: 1 }, - { id: "complete", status: "completed", attempt: 1 }, + { + id: "incomplete", + status: "completed_with_incomplete_coverage", + attempt: 1, + coverage: "partial", + }, + { id: "complete", status: "completed", attempt: 1, coverage: "complete" }, ]); }); }); From 692090d00e6191c3ae3abf126b7e2a8837631e13 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Wed, 5 Aug 2026 21:55:54 +0000 Subject: [PATCH 2/3] test: keep incomplete bulk scan regressions together --- sdk/typescript/tests-ts/cli.test.ts | 104 ---------------------- sdk/typescript/tests-ts/multiscan.test.ts | 84 +++++++++++++++++ 2 files changed, 84 insertions(+), 104 deletions(-) diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index ec8288ba..725e8b42 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -759,110 +759,6 @@ describe("CLI", () => { } }); - test.each(["partial", "unknown"] as const)( - "keeps sealed %s-coverage bulk scans fail-closed without retrying", - async (completeness) => { - const root = await mkdtemp( - join(tmpdir(), "codex-security-cli-multiscan-incomplete-"), - ); - try { - await multiscanInventory(root); - const outputDir = join( - root, - "results", - "artifacts", - "sample", - "attempt-1", - ); - await mkdir(outputDir, { recursive: true }); - await Promise.all( - [ - "scan-manifest.json", - "findings.json", - "coverage.json", - "report.md", - ].map((name) => writeFile(join(outputDir, name), "{}\n")), - ); - const stdout = capture(); - const stderr = capture(); - let attempts = 0; - const arguments_ = [ - "bulk-scan", - "repositories.csv", - "--output-dir", - "results", - "--max-attempts", - "3", - "--json", - ]; - const clientDependencies = dependencies({ - currentDirectory: root, - result: fakeResult([], completeness), - onRun: () => { - attempts += 1; - }, - }); - - expect( - await main( - arguments_, - stdout.stream, - stderr.stream, - clientDependencies, - ), - ).toBe(2); - expect(attempts).toBe(1); - expect(JSON.parse(stdout.text())).toMatchObject({ - total: 1, - completed: 0, - incomplete: 1, - failed: 0, - skipped: 0, - }); - expect(stderr.text()).toContain( - "sample completed_with_incomplete_coverage (attempt 1)", - ); - expect(stderr.text()).toContain( - `Scan coverage is ${completeness}; results may be incomplete.`, - ); - expect(stderr.text()).not.toContain("attempt 2"); - const receipt = JSON.parse( - ( - await readFile(join(root, "results", "results.jsonl"), "utf8") - ).trim(), - ) as Record; - expect(receipt).toMatchObject({ - status: "completed_with_incomplete_coverage", - coverage: completeness, - outputDir, - }); - - const resumedOutput = capture(); - const resumedError = capture(); - expect( - await main( - arguments_, - resumedOutput.stream, - resumedError.stream, - clientDependencies, - ), - ).toBe(2); - expect(JSON.parse(resumedOutput.text())).toMatchObject({ - completed: 0, - incomplete: 1, - failed: 0, - skipped: 1, - }); - expect(resumedError.text()).toContain( - `Scan coverage is ${completeness}; results may be incomplete.`, - ); - expect(attempts).toBe(1); - } finally { - await rm(root, { recursive: true, force: true }); - } - }, - ); - test.each([ [ "OpenRouter", diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index fce082e7..2b28a8d2 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -13,9 +13,11 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test } from "bun:test"; +import { main } from "../src/cli.js"; import type { ScanResult } from "../src/result.js"; import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js"; import { resolveTrustedExecutable } from "../src/trusted-executable.js"; +import { capture, dependencies, fakeResult } from "./cli-fixtures.js"; type MultiscanOptions = Parameters[0]; type SecurityClient = ReturnType; @@ -333,6 +335,88 @@ describe("multiscan", () => { }, ); + test.each(["partial", "unknown"] as const)( + "keeps sealed %s-coverage CLI runs fail-closed without retrying", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, "sample"); + await writeFile( + paths.input, + `id,repository,revision\nsample,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "sample", "attempt-1"); + await completedScan(outputDir, completeness); + const stdout = capture(); + const stderr = capture(); + let attempts = 0; + const arguments_ = [ + "bulk-scan", + "repositories.csv", + "--output-dir", + "results", + "--max-attempts", + "3", + "--json", + ]; + const clientDependencies = dependencies({ + currentDirectory: paths.root, + result: fakeResult([], completeness), + onRun: () => { + attempts += 1; + }, + }); + + expect( + await main( + arguments_, + stdout.stream, + stderr.stream, + clientDependencies, + ), + ).toBe(2); + expect(attempts).toBe(1); + expect(JSON.parse(stdout.text())).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 0, + }); + const warning = `Scan coverage is ${completeness}; results may be incomplete.`; + expect(stderr.text()).toContain( + "sample completed_with_incomplete_coverage (attempt 1)", + ); + expect(stderr.text()).toContain(warning); + expect(stderr.text()).not.toContain("attempt 2"); + expect(await results(join(paths.output, "results.jsonl"))).toMatchObject([ + { + status: "completed_with_incomplete_coverage", + coverage: completeness, + outputDir, + }, + ]); + + const resumedOutput = capture(); + const resumedError = capture(); + expect( + await main( + arguments_, + resumedOutput.stream, + resumedError.stream, + clientDependencies, + ), + ).toBe(2); + expect(JSON.parse(resumedOutput.text())).toMatchObject({ + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(resumedError.text()).toContain(warning); + expect(attempts).toBe(1); + }, + ); + test("retries incomplete scans that are missing required artifacts", async () => { const paths = await fixture(); const source = await repository(paths.root, "missing-artifact"); From ab6c47d103f780fa5a054a8a19146a3d0fcd0cf0 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Thu, 6 Aug 2026 16:56:03 +0000 Subject: [PATCH 3/3] fix: resume legacy sealed incomplete bulk scans --- sdk/typescript/src/multiscan.ts | 41 +++++- sdk/typescript/tests-ts/multiscan.test.ts | 155 ++++++++++++++++++++++ 2 files changed, 189 insertions(+), 7 deletions(-) diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts index de6e8b49..6e967324 100644 --- a/sdk/typescript/src/multiscan.ts +++ b/sdk/typescript/src/multiscan.ts @@ -124,28 +124,33 @@ async function runCampaign( for (const task of tasks) { const receipt = receipts.get(task.id.toLowerCase()); if ( - (receipt?.status === "completed" || - receipt?.status === "completed_with_incomplete_coverage") && + receipt !== undefined && receipt.outputDir === join(output, "artifacts", task.id, `attempt-${receipt.attempt}`) && (await hasArtifacts(receipt.outputDir)) ) { if (receipt.status === "completed") { completed += 1; - } else { + continue; + } + const coverage = + receipt.status === "completed_with_incomplete_coverage" + ? receipt.coverage ?? "unknown" + : await legacyIncompleteCoverage(receipt); + if (coverage !== undefined) { incomplete += 1; options.onProgress?.({ repository: task.id, - status: receipt.status, + status: "completed_with_incomplete_coverage", attempt: receipt.attempt, warning: receipt.warning ?? - `Scan coverage is ${receipt.coverage ?? "unknown"}; results may be incomplete.`, + `Scan coverage is ${coverage}; results may be incomplete.`, }); + continue; } - } else { - pending.push(task); } + pending.push(task); } const skipped = completed + incomplete; if (pending.length === 0) { @@ -394,6 +399,28 @@ async function hasArtifacts(path: string): Promise { } } +async function legacyIncompleteCoverage( + receipt: MultiscanReceipt, +): Promise | undefined> { + if ( + receipt.status !== "failed" || + receipt.error !== "Multiscan repository coverage is incomplete." + ) { + return undefined; + } + try { + const coverage = JSON.parse( + await readFile(join(receipt.outputDir, "coverage.json"), "utf8"), + ) as { completeness?: unknown }; + return coverage.completeness === "partial" || + coverage.completeness === "unknown" + ? coverage.completeness + : undefined; + } catch { + return undefined; + } +} + function parseInventory( source: string, directory: string, diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index 2b28a8d2..d01a1f25 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -335,6 +335,161 @@ describe("multiscan", () => { }, ); + test.each(["partial", "unknown"] as const)( + "resumes legacy sealed %s coverage without rerunning or duplicating cost", + async (completeness) => { + const paths = await fixture(); + const source = await repository(paths.root, `legacy-${completeness}`); + await writeFile( + paths.input, + `id,repository,revision\nlegacy,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1"); + await completedScan(outputDir, completeness); + await writeFile( + join(outputDir, "coverage.json"), + `${JSON.stringify({ completeness })}\n`, + ); + const cost = { + model: "gpt-5.6-sol", + inputTokens: 1_250, + cachedInputTokens: 200, + cacheWriteInputTokens: 0, + outputTokens: 30, + estimatedUsd: 231.73, + }; + const receipt = { + id: "legacy", + repository: source.path, + revision: source.revision, + mode: "standard", + status: "failed", + attempt: 1, + outputDir, + cost, + error: "Multiscan repository coverage is incomplete.", + }; + await writeFile( + join(paths.output, "results.jsonl"), + `${JSON.stringify(receipt)}\n`, + ); + const progress: Parameters< + NonNullable + >[0][] = []; + let attempts = 0; + const security = client(async (_repository, scanOptions = {}) => { + attempts += 1; + return await completedScan(scanOptions.outputDir!); + }); + + const summary = await runMultiscan( + options(paths, security, { + maxAttempts: 3, + onProgress: (event) => progress.push(event), + }), + ); + + expect(summary).toMatchObject({ + total: 1, + completed: 0, + incomplete: 1, + failed: 0, + skipped: 1, + }); + expect(attempts).toBe(0); + expect(progress).toEqual([ + { + repository: "legacy", + status: "completed_with_incomplete_coverage", + attempt: 1, + warning: `Scan coverage is ${completeness}; results may be incomplete.`, + }, + ]); + expect(await results(summary.resultsPath)).toEqual([receipt]); + + await runMultiscan(options(paths, security, { maxAttempts: 3 })); + expect(attempts).toBe(0); + expect(await results(summary.resultsPath)).toEqual([receipt]); + }, + ); + + test.each([ + ["operational failures", "partial", "Worker exited unexpectedly.", false], + [ + "complete coverage", + "complete", + "Multiscan repository coverage is incomplete.", + false, + ], + [ + "malformed coverage", + "malformed", + "Multiscan repository coverage is incomplete.", + false, + ], + [ + "missing artifacts", + "partial", + "Multiscan repository coverage is incomplete.", + true, + ], + ] as const)( + "continues retrying legacy %s", + async (_scenario, completeness, error, missingArtifact) => { + const paths = await fixture(); + const source = await repository(paths.root, "legacy-retry"); + await writeFile( + paths.input, + `id,repository,revision\nlegacy,${source.path},${source.revision}\n`, + ); + const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1"); + await completedScan(outputDir); + await writeFile( + join(outputDir, "coverage.json"), + completeness === "malformed" + ? "{\n" + : `${JSON.stringify({ completeness })}\n`, + ); + if (missingArtifact) await rm(join(outputDir, "report.md")); + await writeFile( + join(paths.output, "results.jsonl"), + `${JSON.stringify({ + id: "legacy", + repository: source.path, + revision: source.revision, + mode: "standard", + status: "failed", + attempt: 1, + outputDir, + error, + })}\n`, + ); + let attempts = 0; + + const summary = await runMultiscan( + options( + paths, + client(async (_repository, scanOptions = {}) => { + attempts += 1; + return await completedScan(scanOptions.outputDir!); + }), + ), + ); + + expect(summary).toMatchObject({ + completed: 1, + incomplete: 0, + failed: 0, + skipped: 0, + }); + expect(attempts).toBe(1); + expect(await results(summary.resultsPath)).toMatchObject([ + { status: "failed", attempt: 1, error }, + { status: "completed", attempt: 2, coverage: "complete" }, + ]); + }, + ); + test.each(["partial", "unknown"] as const)( "keeps sealed %s-coverage CLI runs fail-closed without retrying", async (completeness) => {