diff --git a/scripts/storage/reconcile-stores.ts b/scripts/storage/reconcile-stores.ts index caae2ba24b..3d0aa222b5 100644 --- a/scripts/storage/reconcile-stores.ts +++ b/scripts/storage/reconcile-stores.ts @@ -37,6 +37,12 @@ * heals into normal buckets or silently writes compliance records into an unprotected bucket. * REPORT mode does not require this variable and is unaffected by its value (no-op). * + * Exact-cap heal (`--exact-cap` with `--heal`): production path that binds the heal to both + * the exact additive candidate count (must equal RECONCILE_HEAL_CAP) and a previously + * approved candidate-set digest (RECONCILE_EXPECTED_CANDIDATE_SET_SHA256). Both checks run + * before any WORM probe, download, or PUT. Normal `--heal` keeps max-cap semantics + * (candidate count ≤ RECONCILE_HEAL_CAP / DEFAULT_HEAL_CAP). + * * Very large buckets may need extra heap because all objects are held in memory, e.g.: * node --max-old-space-size=4096 ./node_modules/.bin/ts-node scripts/storage/reconcile-stores.ts * @@ -49,7 +55,10 @@ * --heal / RECONCILE_HEAL=true, ignored by REPORT). Must list every Object-Lock COMPLIANCE * container among the reconciled set; must not list containers outside that set (typo gate). * - Heal: --heal or RECONCILE_HEAL=true - * - Heal cap: RECONCILE_HEAL_CAP (positive integer) or DEFAULT_HEAL_CAP + * - Heal cap: RECONCILE_HEAL_CAP (positive integer) or DEFAULT_HEAL_CAP (normal heal only) + * - Exact-cap heal: --exact-cap (requires --heal / RECONCILE_HEAL=true), plus explicit + * RECONCILE_HEAL_CAP and RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 (64 lowercase hex) + * - Optional report provenance: RECONCILE_API_IMAGE_DIGEST, RECONCILE_OPERATOR_COMMIT_SHA * * Run examples: * RECONCILE_CONTAINERS="kyc support ep2-example" npx ts-node scripts/storage/reconcile-stores.ts @@ -64,7 +73,8 @@ * * Privacy boundary: raw object keys are held in memory for comparison and passed to the * storage SDKs, but are never printed. Verbose diagnostics and heal progress identify an - * object only as a stable SHA-256 key digest. + * object only as a stable SHA-256 key digest. Machine-readable Gate-1 summary lines never + * include raw keys or user metadata. */ import { @@ -90,7 +100,10 @@ import { export const DEFAULT_HEAL_CAP = 1000; /** Runtime marker required by the production operator wrapper before credentials are passed. */ -export const RECONCILER_PRIVACY_LOG_VERSION = 'storage-reconciler-private-logs-v1'; +export const RECONCILER_PRIVACY_LOG_VERSION = 'storage-reconciler-private-logs-v2'; + +/** Schema version for the machine-readable Gate-1 `RECONCILE_REPORT_JSON` line. */ +export const RECONCILE_REPORT_SCHEMA_VERSION = 1; /** * Azure lastModified may legitimately lag S3 (or vice versa) by clock skew / write ordering. @@ -120,6 +133,90 @@ export interface DiffResult { suspectedOverwrite: string[]; } +export type HealDirection = 'azureToS3' | 's3ToAzure'; + +export interface DirectionSummary { + count: number; + bytes: number; +} + +export interface ContainerAdditiveSummary { + container: string; + azureToS3: DirectionSummary; + s3ToAzure: DirectionSummary; +} + +/** Privacy-safe additive candidate (no raw key). */ +export interface AdditiveCandidate { + container: string; + direction: HealDirection; + keySha256: string; + size: number; +} + +export interface AdditiveReportTotals { + onlyOnAzure: number; + onlyOnS3: number; + sizeMismatch: number; + suspectedOverwrite: number; + azureToS3: DirectionSummary; + s3ToAzure: DirectionSummary; + additiveCandidates: number; +} + +/** Canonical additive Gate-1 summary over all containers (privacy-safe). */ +export interface AdditiveReconcileSummary { + containers: ContainerAdditiveSummary[]; + totals: AdditiveReportTotals; + candidateSetSha256: string; +} + +/** Machine-readable Gate-1 report payload (no raw keys / user metadata). */ +export interface MachineReadableReconcileReport { + schemaVersion: number; + privacyLogVersion: string; + mode: 'REPORT' | 'HEAL'; + generatedAtUtc: string; + apiImageDigest?: string; + operatorCommitSha?: string; + containers: ContainerAdditiveSummary[]; + totals: AdditiveReportTotals; + candidateSetSha256: string; +} + +export interface DirectionHealStats { + healedCount: number; + healedBytes: number; + skippedCount: number; + skippedBytes: number; +} + +/** Per-container inventory + diff inputs for additive HEAL orchestration (privacy-safe). */ +export interface HealContainerReport { + container: string; + diff: DiffResult; + azureByKey: ReadonlyMap; + s3ByKey: ReadonlyMap; +} + +/** Inputs for the production additive HEAL path (authorization + sequential copies). */ +export interface AdditiveHealOrchestrationParams { + reports: ReadonlyArray; + s3: S3Client; + azure: Pick; + wormContainers: ReadonlySet; + healCap: number; + exactCap: boolean; + actualCandidateSetSha256: string; + expectedCandidateSetSha256?: string; +} + +/** Directional heal stats returned by runAdditiveHealOrchestration for summary/verification. */ +export interface AdditiveHealOrchestrationStats { + azureToS3: DirectionHealStats; + s3ToAzure: DirectionHealStats; +} + // Unit-testable pure helpers — no I/O. // --- PURE DIFF / GUARDS --- // @@ -194,6 +291,31 @@ export function assertWithinHealCap(candidateCount: number, cap: number): void { } } +/** + * Exact-cap production binding: additive candidate count must equal RECONCILE_HEAL_CAP and the + * computed candidate-set digest must equal the previously approved digest. Fail-closed before + * any heal I/O (WORM probe, download, PUT). + */ +export function assertExactHealBinding( + additiveCandidateCount: number, + healCap: number, + actualCandidateSetSha256: string, + expectedCandidateSetSha256: string, +): void { + if (additiveCandidateCount !== healCap) { + throw new Error( + `Exact-cap heal binding failed: additive candidate count ${additiveCandidateCount} ` + + `!== RECONCILE_HEAL_CAP ${healCap}. Refusing heal before any WORM probe, download, or PUT.`, + ); + } + if (actualCandidateSetSha256 !== expectedCandidateSetSha256) { + throw new Error( + `Exact-cap heal binding failed: candidateSetSha256 ${actualCandidateSetSha256} ` + + `!== expected ${expectedCandidateSetSha256}. Refusing heal before any WORM probe, download, or PUT.`, + ); + } +} + /** Fail if any requested container name is missing from the S3 bucket inventory. */ export function assertRequestedContainersExist(existingS3Buckets: string[], requested: string[]): void { const existing = new Set(existingS3Buckets); @@ -379,6 +501,198 @@ export function assertWormContainersConfig( } } +// --- ADDITIVE SUMMARY / DIGEST (privacy-safe, pure) --- // + +export function hashObjectKeySha256(key: string): string { + return crypto.createHash('sha256').update(key, 'utf8').digest('hex'); +} + +export function indexStoredObjectsByKey(objects: readonly StoredObject[]): Map { + const map = new Map(); + for (const obj of objects) { + map.set(obj.key, obj); + } + return map; +} + +export function sumBytesForKeys(keys: readonly string[], byKey: ReadonlyMap): number { + let bytes = 0; + for (const key of keys) { + const obj = byKey.get(key); + if (!obj) { + throw new Error( + `Missing inventory object for key-sha256=${hashObjectKeySha256(key)} while summing candidate bytes`, + ); + } + bytes += obj.size; + } + return bytes; +} + +export function buildDirectionSummary( + keys: readonly string[], + byKey: ReadonlyMap, +): DirectionSummary { + return { count: keys.length, bytes: sumBytesForKeys(keys, byKey) }; +} + +/** + * Collect privacy-safe additive candidates for one container. Only onlyOnAzure / onlyOnS3 + * contribute; sizeMismatch and suspectedOverwrite are never additive heal candidates. + */ +export function collectAdditiveCandidatesForContainer( + container: string, + diff: DiffResult, + azureByKey: ReadonlyMap, + s3ByKey: ReadonlyMap, +): AdditiveCandidate[] { + const candidates: AdditiveCandidate[] = []; + + for (const key of diff.onlyOnAzure) { + const obj = azureByKey.get(key); + if (!obj) { + throw new Error(`Missing Azure inventory object for ${safeObjectReference(container, key)}`); + } + candidates.push({ + container, + direction: 'azureToS3', + keySha256: hashObjectKeySha256(key), + size: obj.size, + }); + } + + for (const key of diff.onlyOnS3) { + const obj = s3ByKey.get(key); + if (!obj) { + throw new Error(`Missing S3 inventory object for ${safeObjectReference(container, key)}`); + } + candidates.push({ + container, + direction: 's3ToAzure', + keySha256: hashObjectKeySha256(key), + size: obj.size, + }); + } + + return candidates; +} + +/** Canonical sort for candidate-set digest (order-independent over the multiset). */ +export function compareAdditiveCandidates(a: AdditiveCandidate, b: AdditiveCandidate): number { + if (a.container !== b.container) return a.container < b.container ? -1 : 1; + if (a.direction !== b.direction) return a.direction < b.direction ? -1 : 1; + if (a.keySha256 !== b.keySha256) return a.keySha256 < b.keySha256 ? -1 : 1; + if (a.size !== b.size) return a.size < b.size ? -1 : 1; + return 0; +} + +/** + * Deterministic, order-independent digest of the exact additive candidate set. + * Each entry is container, direction, sha256(key), size — never a raw key. + */ +export function computeCandidateSetSha256(candidates: readonly AdditiveCandidate[]): string { + const sorted = [...candidates].sort(compareAdditiveCandidates); + // Fixed field order; array form avoids object key-order ambiguity. + const canonical = JSON.stringify(sorted.map((c) => [c.container, c.direction, c.keySha256, c.size])); + return crypto.createHash('sha256').update(canonical, 'utf8').digest('hex'); +} + +export function buildContainerAdditiveSummary( + container: string, + diff: DiffResult, + azureByKey: ReadonlyMap, + s3ByKey: ReadonlyMap, +): ContainerAdditiveSummary { + return { + container, + azureToS3: buildDirectionSummary(diff.onlyOnAzure, azureByKey), + s3ToAzure: buildDirectionSummary(diff.onlyOnS3, s3ByKey), + }; +} + +export function buildAdditiveReconcileSummary( + containerReports: ReadonlyArray<{ + container: string; + diff: DiffResult; + azureObjs: readonly StoredObject[]; + s3Objs: readonly StoredObject[]; + }>, +): AdditiveReconcileSummary { + const containers: ContainerAdditiveSummary[] = []; + const allCandidates: AdditiveCandidate[] = []; + let onlyOnAzure = 0; + let onlyOnS3 = 0; + let sizeMismatch = 0; + let suspectedOverwrite = 0; + let azureToS3Count = 0; + let azureToS3Bytes = 0; + let s3ToAzureCount = 0; + let s3ToAzureBytes = 0; + + for (const report of containerReports) { + const azureByKey = indexStoredObjectsByKey(report.azureObjs); + const s3ByKey = indexStoredObjectsByKey(report.s3Objs); + const summary = buildContainerAdditiveSummary(report.container, report.diff, azureByKey, s3ByKey); + containers.push(summary); + allCandidates.push(...collectAdditiveCandidatesForContainer(report.container, report.diff, azureByKey, s3ByKey)); + + onlyOnAzure += report.diff.onlyOnAzure.length; + onlyOnS3 += report.diff.onlyOnS3.length; + sizeMismatch += report.diff.sizeMismatch.length; + suspectedOverwrite += report.diff.suspectedOverwrite.length; + azureToS3Count += summary.azureToS3.count; + azureToS3Bytes += summary.azureToS3.bytes; + s3ToAzureCount += summary.s3ToAzure.count; + s3ToAzureBytes += summary.s3ToAzure.bytes; + } + + const additiveCandidates = azureToS3Count + s3ToAzureCount; + return { + containers, + totals: { + onlyOnAzure, + onlyOnS3, + sizeMismatch, + suspectedOverwrite, + azureToS3: { count: azureToS3Count, bytes: azureToS3Bytes }, + s3ToAzure: { count: s3ToAzureCount, bytes: s3ToAzureBytes }, + additiveCandidates, + }, + candidateSetSha256: computeCandidateSetSha256(allCandidates), + }; +} + +export function buildMachineReadableReconcileReport( + summary: AdditiveReconcileSummary, + mode: 'REPORT' | 'HEAL', + generatedAtUtc: string, + provenance?: { apiImageDigest?: string; operatorCommitSha?: string }, +): MachineReadableReconcileReport { + const report: MachineReadableReconcileReport = { + schemaVersion: RECONCILE_REPORT_SCHEMA_VERSION, + privacyLogVersion: RECONCILER_PRIVACY_LOG_VERSION, + mode, + generatedAtUtc, + containers: summary.containers, + totals: summary.totals, + candidateSetSha256: summary.candidateSetSha256, + }; + + if (provenance?.apiImageDigest !== undefined) { + report.apiImageDigest = provenance.apiImageDigest; + } + if (provenance?.operatorCommitSha !== undefined) { + report.operatorCommitSha = provenance.operatorCommitSha; + } + + return report; +} + +/** Exactly one machine-readable line for operator / Gate-1 automation. */ +export function formatReconcileReportJsonLine(report: MachineReadableReconcileReport): string { + return `RECONCILE_REPORT_JSON=${JSON.stringify(report)}`; +} + // size/lastModified from list pages only — no HeadObject per key. // --- LISTING --- // @@ -479,6 +793,8 @@ function dedupePreserveOrder(items: string[]): string[] { return out; } +const EXPECTED_CANDIDATE_SET_SHA256_RE = /^[a-f0-9]{64}$/; + export function parseConfig(): { containers: string[]; ignoreBuckets: string[]; @@ -486,17 +802,26 @@ export function parseConfig(): { heal: boolean; healCap: number; verbose: boolean; + exactCap: boolean; + expectedCandidateSetSha256?: string; + apiImageDigest?: string; + operatorCommitSha?: string; } { const args = process.argv.slice(2); for (const a of args) { - if (a.startsWith('--') && a !== '--heal' && a !== '--verbose') { + if (a.startsWith('--') && a !== '--heal' && a !== '--verbose' && a !== '--exact-cap') { throw new Error(`Unknown flag: ${a}`); } } const heal = args.includes('--heal') || process.env.RECONCILE_HEAL === 'true'; const verbose = args.includes('--verbose'); + const exactCap = args.includes('--exact-cap'); + + if (exactCap && !heal) { + throw new Error('--exact-cap is only allowed together with --heal (or RECONCILE_HEAL=true)'); + } let containers = args.filter((a) => !a.startsWith('--')); if (containers.length === 0) { @@ -526,9 +851,43 @@ export function parseConfig(): { throw new Error(`Invalid RECONCILE_HEAL_CAP: ${rawCap} (expected positive integer)`); } healCap = n; + } else if (exactCap) { + throw new Error( + 'RECONCILE_HEAL_CAP is required when --exact-cap is set (exact additive candidate count binding; no silent default)', + ); } - return { containers, ignoreBuckets, wormContainers, heal, healCap, verbose }; + let expectedCandidateSetSha256: string | undefined; + if (exactCap) { + const expected = process.env.RECONCILE_EXPECTED_CANDIDATE_SET_SHA256; + if (expected === undefined || expected === '') { + throw new Error( + 'RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 is required when --exact-cap is set and must be exactly 64 lowercase hex characters', + ); + } + if (!EXPECTED_CANDIDATE_SET_SHA256_RE.test(expected)) { + throw new Error( + 'RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 must be exactly 64 lowercase hex characters when --exact-cap is set', + ); + } + expectedCandidateSetSha256 = expected; + } + + const apiImageDigest = process.env.RECONCILE_API_IMAGE_DIGEST; + const operatorCommitSha = process.env.RECONCILE_OPERATOR_COMMIT_SHA; + + return { + containers, + ignoreBuckets, + wormContainers, + heal, + healCap, + verbose, + exactCap, + ...(expectedCandidateSetSha256 !== undefined ? { expectedCandidateSetSha256 } : {}), + ...(apiImageDigest !== undefined && apiImageDigest !== '' ? { apiImageDigest } : {}), + ...(operatorCommitSha !== undefined && operatorCommitSha !== '' ? { operatorCommitSha } : {}), + }; } function isEmptyDiff(diff: DiffResult): boolean { @@ -545,15 +904,66 @@ function countAdditive(diff: DiffResult): number { } export function safeObjectReference(container: string, key: string): string { - const keyDigest = crypto.createHash('sha256').update(key).digest('hex'); + const keyDigest = hashObjectKeySha256(key); return `${container}/key-sha256:${keyDigest}`; } -export function printCategory(container: string, label: string, keys: string[], verbose: boolean): void { - console.log(` ${label}: ${keys.length}`); +/** + * Privacy-safe detail line for --verbose: fixed technical allowlist only — container, + * sha256(key), source, size, UTC lastModified. Never ETag, content hashes, raw keys, + * content-type, or user metadata. + */ +export function formatDetailObjectLine( + container: string, + key: string, + source: 'azure' | 's3', + obj: StoredObject, +): string { + const keySha256 = hashObjectKeySha256(key); + const lastModifiedUtc = obj.lastModified.toISOString(); + return ` - container=${container} key-sha256=${keySha256} source=${source} size=${obj.size} lastModified=${lastModifiedUtc}`; +} + +export function printCategory( + container: string, + label: string, + keys: string[], + verbose: boolean, + options?: { + bytes?: number; + azureByKey?: ReadonlyMap; + s3ByKey?: ReadonlyMap; + dualSide?: boolean; + singleSource?: 'azure' | 's3'; + }, +): void { + const bytesSuffix = options?.bytes !== undefined ? ` (bytes=${options.bytes})` : ''; + console.log(` ${label}: ${keys.length}${bytesSuffix}`); if (!verbose || keys.length === 0) return; + const samples = keys.slice(0, 20); - for (const key of samples) console.log(` - ${safeObjectReference(container, key)}`); + for (const key of samples) { + if (options?.dualSide) { + const azureObj = options.azureByKey?.get(key); + const s3Obj = options.s3ByKey?.get(key); + if (!azureObj || !s3Obj) { + throw new Error(`Missing dual-side inventory for ${safeObjectReference(container, key)} in detail mode`); + } + console.log(formatDetailObjectLine(container, key, 'azure', azureObj)); + console.log(formatDetailObjectLine(container, key, 's3', s3Obj)); + } else { + const source = options?.singleSource; + if (source === undefined) { + throw new Error(`Detail mode requires singleSource or dualSide for category ${label}`); + } + const byKey = source === 'azure' ? options.azureByKey : options.s3ByKey; + const obj = byKey?.get(key); + if (!obj) { + throw new Error(`Missing ${source} inventory for ${safeObjectReference(container, key)} in detail mode`); + } + console.log(formatDetailObjectLine(container, key, source, obj)); + } + } if (keys.length > 20) console.log(` ... and ${keys.length - 20} more`); } @@ -593,14 +1003,53 @@ export function isAzurePreconditionFailed(err: unknown): boolean { return e?.statusCode === 412 || e?.details?.errorCode === 'BlobAlreadyExists'; } +/** + * Fail-closed gate for the digest-bound inventory size passed into heal copies. + * Rejects non-integers, negatives, and non-safe integers before any target write. + */ +export function assertExpectedSourceSize( + expectedSourceSize: number, + direction: 'azure->s3' | 's3->azure', + objectRef: string, +): void { + if (!Number.isSafeInteger(expectedSourceSize) || expectedSourceSize < 0) { + throw new Error( + `Invalid expectedSourceSize for ${direction} copy of ${objectRef}: ` + + `expected non-negative safe integer, got ${String(expectedSourceSize)}`, + ); + } +} + +/** + * After a full source download, refuse to write if the loaded buffer length differs from the + * inventory size that was bound into the candidate set (container + direction + sha256(key) + size). + * Closes the race where the source object changes size between inventory and download. + */ +export function assertDownloadedSourceSize( + actualSize: number, + expectedSourceSize: number, + direction: 'azure->s3' | 's3->azure', + objectRef: string, +): void { + if (actualSize !== expectedSourceSize) { + throw new Error( + `Source size race for ${direction}: expected ${expectedSourceSize} bytes, ` + + `got ${actualSize} bytes for ${objectRef}`, + ); + } +} + export async function copyAzureToS3( azureContainer: ContainerClient, s3: S3Client, bucket: string, key: string, + expectedSourceSize: number, ): Promise<'healed' | 'skipped'> { const blobClient = azureContainer.getBlockBlobClient(key); const objectRef = safeObjectReference(bucket, key); + assertExpectedSourceSize(expectedSourceSize, 'azure->s3', objectRef); + const download = await blobClient.download().catch((err) => { throw new Error(`Azure download failed for ${objectRef}`, { cause: err }); }); @@ -614,6 +1063,9 @@ export async function copyAzureToS3( throw new Error(`Azure download stream failed for ${objectRef}`, { cause: err }); } + // Inventory size is digest-bound; refuse any source that changed size before the conditional PUT. + assertDownloadedSourceSize(data.length, expectedSourceSize, 'azure->s3', objectRef); + try { await s3.send( new PutObjectCommand({ @@ -641,8 +1093,11 @@ export async function copyS3ToAzure( bucket: string, azureContainer: ContainerClient, key: string, + expectedSourceSize: number, ): Promise<'healed' | 'skipped'> { const objectRef = safeObjectReference(bucket, key); + assertExpectedSourceSize(expectedSourceSize, 's3->azure', objectRef); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })).catch((err) => { throw new Error(`S3 download failed for ${objectRef}`, { cause: err }); }); @@ -654,6 +1109,10 @@ export async function copyS3ToAzure( } catch (err) { throw new Error(`S3 download stream failed for ${objectRef}`, { cause: err }); } + + // Inventory size is digest-bound; refuse any source that changed size before the conditional upload. + assertDownloadedSourceSize(data.length, expectedSourceSize, 's3->azure', objectRef); + try { await azureContainer.getBlockBlobClient(key).uploadData(data, { blobHTTPHeaders: { blobContentType: res.ContentType }, @@ -671,10 +1130,114 @@ export async function copyS3ToAzure( return 'healed'; } +/** + * Production additive HEAL orchestration: authorize exact-cap / max-cap binding first, then + * run sequential azure→s3 / s3→azure copies with conditional target writes and WORM + * fail-closed checks. No deletes or overwrites. Authorization runs before any WORM probe, + * download, PUT/upload, or per-container copy loop — including when the live candidate set + * is empty (exact-cap must not short-circuit to a misleading clean success). + */ +export async function runAdditiveHealOrchestration( + params: AdditiveHealOrchestrationParams, +): Promise { + const { + reports, + s3, + azure, + wormContainers, + healCap, + exactCap, + actualCandidateSetSha256, + expectedCandidateSetSha256, + } = params; + + const totalAdditive = reports.reduce((sum, r) => sum + countAdditive(r.diff), 0); + + if (exactCap) { + if (expectedCandidateSetSha256 === undefined) { + throw new Error('Internal error: exactCap set without expectedCandidateSetSha256'); + } + assertExactHealBinding(totalAdditive, healCap, actualCandidateSetSha256, expectedCandidateSetSha256); + } else { + assertWithinHealCap(totalAdditive, healCap); + } + + const wormVerified = new Map(); + const noWormVerified = new Map(); + const azureToS3Stats: DirectionHealStats = { + healedCount: 0, + healedBytes: 0, + skippedCount: 0, + skippedBytes: 0, + }; + const s3ToAzureStats: DirectionHealStats = { + healedCount: 0, + healedBytes: 0, + skippedCount: 0, + skippedBytes: 0, + }; + + for (const r of reports) { + try { + const azureContainer = azure.getContainerClient(r.container); + + for (const key of r.diff.onlyOnAzure) { + const size = r.azureByKey.get(key)?.size; + if (size === undefined) { + throw new Error(`Missing Azure size for ${safeObjectReference(r.container, key)}`); + } + await assertBucketWormIfDeclared(s3, r.container, wormContainers, wormVerified, noWormVerified); + // Pass digest-bound inventory size so copy fails closed if the source changed after inventory. + const result = await copyAzureToS3(azureContainer, s3, r.container, key, size); + if (result === 'healed') { + azureToS3Stats.healedCount++; + azureToS3Stats.healedBytes += size; + logObjectAction('HEALED', 'azure->s3', r.container, key); + } else { + azureToS3Stats.skippedCount++; + azureToS3Stats.skippedBytes += size; + } + } + + for (const key of r.diff.onlyOnS3) { + const size = r.s3ByKey.get(key)?.size; + if (size === undefined) { + throw new Error(`Missing S3 size for ${safeObjectReference(r.container, key)}`); + } + // Pass digest-bound inventory size so copy fails closed if the source changed after inventory. + const result = await copyS3ToAzure(s3, r.container, azureContainer, key, size); + if (result === 'healed') { + s3ToAzureStats.healedCount++; + s3ToAzureStats.healedBytes += size; + logObjectAction('HEALED', 's3->azure', r.container, key); + } else { + s3ToAzureStats.skippedCount++; + s3ToAzureStats.skippedBytes += size; + } + } + } catch (e) { + throw new Error(`[container="${r.container}"] ${e?.message ?? e}`, { cause: e }); + } + } + + return { azureToS3: azureToS3Stats, s3ToAzure: s3ToAzureStats }; +} + // --- MAIN --- // async function main(): Promise { - const { containers, ignoreBuckets, wormContainers, heal, healCap, verbose } = parseConfig(); + const { + containers, + ignoreBuckets, + wormContainers, + heal, + healCap, + verbose, + exactCap, + expectedCandidateSetSha256, + apiImageDigest, + operatorCommitSha, + } = parseConfig(); const s3 = buildS3Client(); const azure = buildAzureClient(); const wormSet = new Set(wormContainers); @@ -684,6 +1247,7 @@ async function main(): Promise { `ignoreBuckets=[${ignoreBuckets.join(', ')}] ` + `wormContainers=[${wormContainers.join(', ')}] mode=${heal ? 'HEAL' : 'REPORT'} ` + `healCap=${healCap}${process.env.RECONCILE_HEAL_CAP === undefined ? ` (DEFAULT_HEAL_CAP)` : ''} ` + + `exactCap=${exactCap} ` + `skewToleranceMs=${OVERWRITE_SKEW_TOLERANCE_MS}`, ); @@ -707,7 +1271,16 @@ async function main(): Promise { assertRequestedContainersExist(existingS3Buckets, containers); assertBucketsAccounted(existingS3Buckets, containers, ignoreBuckets); - type ContainerReport = { container: string; diff: DiffResult; azureCount: number; s3Count: number }; + type ContainerReport = { + container: string; + diff: DiffResult; + azureCount: number; + s3Count: number; + azureObjs: StoredObject[]; + s3Objs: StoredObject[]; + azureByKey: Map; + s3ByKey: Map; + }; const reports: ContainerReport[] = []; for (const container of containers) { @@ -719,109 +1292,138 @@ async function main(): Promise { assertNotOneSidedEmpty(azureObjs.length, s3Objs.length, container); const diff = diffStores(azureObjs, s3Objs, OVERWRITE_SKEW_TOLERANCE_MS); - reports.push({ container, diff, azureCount: azureObjs.length, s3Count: s3Objs.length }); + const azureByKey = indexStoredObjectsByKey(azureObjs); + const s3ByKey = indexStoredObjectsByKey(s3Objs); + reports.push({ + container, + diff, + azureCount: azureObjs.length, + s3Count: s3Objs.length, + azureObjs, + s3Objs, + azureByKey, + s3ByKey, + }); + + const azureToS3 = buildDirectionSummary(diff.onlyOnAzure, azureByKey); + const s3ToAzure = buildDirectionSummary(diff.onlyOnS3, s3ByKey); console.log(`\n[${container}] azure=${azureObjs.length} s3=${s3Objs.length}`); - printCategory(container, 'onlyOnAzure', diff.onlyOnAzure, verbose); - printCategory(container, 'onlyOnS3', diff.onlyOnS3, verbose); - printCategory(container, 'sizeMismatch', diff.sizeMismatch, verbose); - printCategory(container, 'suspectedOverwrite', diff.suspectedOverwrite, verbose); + printCategory(container, 'onlyOnAzure', diff.onlyOnAzure, verbose, { + bytes: azureToS3.bytes, + azureByKey, + singleSource: 'azure', + }); + printCategory(container, 'onlyOnS3', diff.onlyOnS3, verbose, { + bytes: s3ToAzure.bytes, + s3ByKey, + singleSource: 's3', + }); + printCategory(container, 'sizeMismatch', diff.sizeMismatch, verbose, { + azureByKey, + s3ByKey, + dualSide: true, + }); + printCategory(container, 'suspectedOverwrite', diff.suspectedOverwrite, verbose, { + azureByKey, + s3ByKey, + dualSide: true, + }); + console.log( + ` candidateBytes: azureToS3=${azureToS3.bytes} s3ToAzure=${s3ToAzure.bytes} ` + + `(counts azureToS3=${azureToS3.count} s3ToAzure=${s3ToAzure.count})`, + ); if (isEmptyDiff(diff)) console.log(' OK: no divergence'); } catch (e) { throw new Error(`[container="${container}"] ${e?.message ?? e}`, { cause: e }); } } - const totals = { - onlyOnAzure: 0, - onlyOnS3: 0, - sizeMismatch: 0, - suspectedOverwrite: 0, - }; - for (const r of reports) { - totals.onlyOnAzure += r.diff.onlyOnAzure.length; - totals.onlyOnS3 += r.diff.onlyOnS3.length; - totals.sizeMismatch += r.diff.sizeMismatch.length; - totals.suspectedOverwrite += r.diff.suspectedOverwrite.length; - } + const additiveSummary = buildAdditiveReconcileSummary(reports); + const totals = additiveSummary.totals; console.log( `\nAGGREGATE across ${containers.length} container(s): ` + `onlyOnAzure=${totals.onlyOnAzure} onlyOnS3=${totals.onlyOnS3} ` + - `sizeMismatch=${totals.sizeMismatch} suspectedOverwrite=${totals.suspectedOverwrite}`, + `sizeMismatch=${totals.sizeMismatch} suspectedOverwrite=${totals.suspectedOverwrite} ` + + `azureToS3.count=${totals.azureToS3.count} azureToS3.bytes=${totals.azureToS3.bytes} ` + + `s3ToAzure.count=${totals.s3ToAzure.count} s3ToAzure.bytes=${totals.s3ToAzure.bytes} ` + + `additiveCandidates=${totals.additiveCandidates} ` + + `candidateSetSha256=${additiveSummary.candidateSetSha256}`, ); + const machineReport = buildMachineReadableReconcileReport( + additiveSummary, + heal ? 'HEAL' : 'REPORT', + new Date().toISOString(), + { + ...(apiImageDigest !== undefined ? { apiImageDigest } : {}), + ...(operatorCommitSha !== undefined ? { operatorCommitSha } : {}), + }, + ); + // Exactly one machine-readable line before REPORT / HEAL decision. + console.log(formatReconcileReportJsonLine(machineReport)); + const anyGateBlocking = reports.some((r) => isGateBlocking(r.diff)); - if (!anyGateBlocking) { - for (const r of reports) { - if (r.diff.suspectedOverwrite.length > 0) { - console.log( - `ADVISORY: suspectedOverwrite=${r.diff.suspectedOverwrite.length} in ${r.container} ` + - `(lastModified hint; unreliable after heals -- NOTE: (key,size) parity only; byte identity ` + - `is NOT verified here. Same-size content divergence is out of scope and requires a separate ` + - `byte-level Azure/S3 comparison (not part of this tool) before Azure teardown; does NOT block the gate)`, - ); + + // REPORT: unchanged early clean success / divergence exit. HEAL always enters orchestration + // (including zero candidates) so exact-cap cannot bypass authorization via clean parity. + if (!heal) { + if (!anyGateBlocking) { + for (const r of reports) { + if (r.diff.suspectedOverwrite.length > 0) { + console.log( + `ADVISORY: suspectedOverwrite=${r.diff.suspectedOverwrite.length} in ${r.container} ` + + `(lastModified hint; unreliable after heals -- NOTE: (key,size) parity only; byte identity ` + + `is NOT verified here. Same-size content divergence is out of scope and requires a separate ` + + `byte-level Azure/S3 comparison (not part of this tool) before Azure teardown; does NOT block the gate)`, + ); + } } + console.log( + `RECONCILED: 0 (key,size) divergence across ${containers.length} containers ` + + `(hard gate is (key,size) parity; suspectedOverwrite is an advisory lastModified hint, ` + + `unreliable after heals) -- NOTE: (key,size) parity only; byte identity is NOT verified here. ` + + `Same-size content divergence is out of scope and requires a separate byte-level Azure/S3 ` + + `comparison (not part of this tool) before Azure teardown.`, + ); + return 0; } - console.log( - `RECONCILED: 0 (key,size) divergence across ${containers.length} containers ` + - `(hard gate is (key,size) parity; suspectedOverwrite is an advisory lastModified hint, ` + - `unreliable after heals) -- NOTE: (key,size) parity only; byte identity is NOT verified here. ` + - `Same-size content divergence is out of scope and requires a separate byte-level Azure/S3 ` + - `comparison (not part of this tool) before Azure teardown.`, - ); - return 0; - } - if (!heal) { console.error( `DIVERGENCE: not reconciled across ${containers.length} container(s) ` + `(onlyOnAzure=${totals.onlyOnAzure}, onlyOnS3=${totals.onlyOnS3}, ` + - `sizeMismatch=${totals.sizeMismatch}, suspectedOverwrite=${totals.suspectedOverwrite}). ` + + `sizeMismatch=${totals.sizeMismatch}, suspectedOverwrite=${totals.suspectedOverwrite}, ` + + `azureToS3.bytes=${totals.azureToS3.bytes}, s3ToAzure.bytes=${totals.s3ToAzure.bytes}, ` + + `candidateSetSha256=${additiveSummary.candidateSetSha256}). ` + `Re-run with --heal to copy missing objects only; sizeMismatch/suspectedOverwrite are never auto-healed.`, ); return 1; } - // HEAL: additive copies only (never delete; never overwrite content divergence) - const totalAdditive = reports.reduce((sum, r) => sum + countAdditive(r.diff), 0); - assertWithinHealCap(totalAdditive, healCap); - - const wormVerified = new Map(); - const noWormVerified = new Map(); - let healedCount = 0; - let skippedCount = 0; - - for (const r of reports) { - try { - const azureContainer = azure.getContainerClient(r.container); - - for (const key of r.diff.onlyOnAzure) { - await assertBucketWormIfDeclared(s3, r.container, wormSet, wormVerified, noWormVerified); - const result = await copyAzureToS3(azureContainer, s3, r.container, key); - if (result === 'healed') { - healedCount++; - logObjectAction('HEALED', 'azure->s3', r.container, key); - } else { - skippedCount++; - } - } - - for (const key of r.diff.onlyOnS3) { - const result = await copyS3ToAzure(s3, r.container, azureContainer, key); - if (result === 'healed') { - healedCount++; - logObjectAction('HEALED', 's3->azure', r.container, key); - } else { - skippedCount++; - } - } - } catch (e) { - throw new Error(`[container="${r.container}"] ${e?.message ?? e}`, { cause: e }); - } - } + // HEAL: production orchestration (auth first, then additive copies only) + const healStats = await runAdditiveHealOrchestration({ + reports, + s3, + azure, + wormContainers: wormSet, + healCap, + exactCap, + actualCandidateSetSha256: additiveSummary.candidateSetSha256, + ...(expectedCandidateSetSha256 !== undefined ? { expectedCandidateSetSha256 } : {}), + }); + const azureToS3Stats = healStats.azureToS3; + const s3ToAzureStats = healStats.s3ToAzure; - console.log(`Heal summary: healed=${healedCount} skipped=${skippedCount}`); + const healedCount = azureToS3Stats.healedCount + s3ToAzureStats.healedCount; + const skippedCount = azureToS3Stats.skippedCount + s3ToAzureStats.skippedCount; + console.log( + `Heal summary: healed=${healedCount} skipped=${skippedCount} ` + + `azureToS3.healed.count=${azureToS3Stats.healedCount} azureToS3.healed.bytes=${azureToS3Stats.healedBytes} ` + + `azureToS3.skipped.count=${azureToS3Stats.skippedCount} azureToS3.skipped.bytes=${azureToS3Stats.skippedBytes} ` + + `s3ToAzure.healed.count=${s3ToAzureStats.healedCount} s3ToAzure.healed.bytes=${s3ToAzureStats.healedBytes} ` + + `s3ToAzure.skipped.count=${s3ToAzureStats.skippedCount} s3ToAzure.skipped.bytes=${s3ToAzureStats.skippedBytes}`, + ); // Verification re-diff: additive sides must now be empty; residual content divergence stays red. let residualMismatch = 0; diff --git a/src/integration/infrastructure/storage/__tests__/reconcile-stores.spec.ts b/src/integration/infrastructure/storage/__tests__/reconcile-stores.spec.ts index 493b1cd0a1..cf7cde908c 100644 --- a/src/integration/infrastructure/storage/__tests__/reconcile-stores.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/reconcile-stores.spec.ts @@ -1,20 +1,37 @@ -import { GetObjectLockConfigurationCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; +import { + GetObjectCommand, + GetObjectLockConfigurationCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { mockClient } from 'aws-sdk-client-mock'; +import { Readable } from 'stream'; import { assertBucketWorm, assertBucketWormIfDeclared, assertBucketsAccounted, + assertExactHealBinding, assertNotOneSidedEmpty, assertPageComplete, assertRequestedContainersExist, assertUndeclaredBucketHasNoWorm, assertWithinHealCap, assertWormContainersConfig, + buildAdditiveReconcileSummary, + buildMachineReadableReconcileReport, + collectAdditiveCandidatesForContainer, + computeCandidateSetSha256, copyAzureToS3, copyS3ToAzure, DEFAULT_HEAL_CAP, diffStores, DiffResult, + formatDetailObjectLine, + formatReconcileReportJsonLine, + hashObjectKeySha256, + HealContainerReport, + indexStoredObjectsByKey, isAzurePreconditionFailed, isGateBlocking, isS3PreconditionFailed, @@ -24,7 +41,9 @@ import { OVERWRITE_SKEW_TOLERANCE_MS, parseConfig, printCategory, + RECONCILE_REPORT_SCHEMA_VERSION, RECONCILER_PRIVACY_LOG_VERSION, + runAdditiveHealOrchestration, safeObjectReference, StoredObject, } from '../../../../../scripts/storage/reconcile-stores'; @@ -44,6 +63,17 @@ function storedObject(key: string, size: number, lastModified: Date): StoredObje return { key, size, lastModified }; } +/** Conspicuous ETag-like sentinels — must never appear in privacy-safe detail/logs. */ +const ETAG_SENTINEL_AZURE = '"ETAG_SENTINEL_AZURE_NEVER_LOG_0xDEAD"'; +const ETAG_SENTINEL_S3 = '"ETAG_SENTINEL_S3_NEVER_LOG_abc123"'; + +function assertNoEtagSentinelLeak(text: string): void { + expect(text).not.toContain(ETAG_SENTINEL_AZURE); + expect(text).not.toContain(ETAG_SENTINEL_S3); + expect(text).not.toContain('ETAG_SENTINEL'); + expect(text).not.toMatch(/\betag=/i); +} + async function rejectedError(promise: Promise): Promise { try { await promise; @@ -57,47 +87,74 @@ async function rejectedError(promise: Promise): Promise { const t0 = new Date('2024-01-01T00:00:00.000Z'); const skewMs = 60 * 60 * 1000; // 1 hour — matches OVERWRITE_SKEW_TOLERANCE_MS +/** Conspicuous sentinel raw keys — must never appear in logs, JSON, or digests. */ +const SENTINEL_KEY_A = 'SENTINEL_RAW_KEY_user/999/private-document-NEVER-LOG.pdf'; +const SENTINEL_KEY_B = 'SENTINEL_RAW_KEY_user/888/another-secret-file-NEVER-LOG.bin'; +const SENTINEL_KEY_C = 'SENTINEL_RAW_KEY_acct/777/third-private-NEVER-LOG.dat'; + +function assertNoSentinelLeak(text: string): void { + expect(text).not.toContain(SENTINEL_KEY_A); + expect(text).not.toContain(SENTINEL_KEY_B); + expect(text).not.toContain(SENTINEL_KEY_C); + expect(text).not.toContain('private-document-NEVER-LOG'); + expect(text).not.toContain('another-secret-file-NEVER-LOG'); + expect(text).not.toContain('third-private-NEVER-LOG'); + expect(text).not.toContain('SENTINEL_RAW_KEY'); +} + describe('safeObjectReference', () => { afterEach(() => jest.restoreAllMocks()); it('uses a stable SHA-256 digest without exposing the raw key', () => { - const rawKey = 'user/123/private-document.pdf'; + const rawKey = SENTINEL_KEY_A; const reference = safeObjectReference('kyc', rawKey); expect(reference).toMatch(/^kyc\/key-sha256:[a-f0-9]{64}$/); expect(reference).toBe(safeObjectReference('kyc', rawKey)); expect(reference).not.toContain(rawKey); - expect(reference).not.toContain('private-document'); + assertNoSentinelLeak(reference); }); it('keeps verbose, heal, skip, and object-specific errors free of raw keys', async () => { - const rawKey = 'user/123/private-document.pdf'; + const rawKey = SENTINEL_KEY_A; const expectedReference = safeObjectReference('kyc', rawKey); const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); - printCategory('kyc', 'onlyOnAzure', [rawKey], true); + const azureByKey = indexStoredObjectsByKey([storedObject(rawKey, 42, t0)]); + printCategory('kyc', 'onlyOnAzure', [rawKey], true, { + bytes: 42, + azureByKey, + singleSource: 'azure', + }); logObjectAction('HEALED', 'azure->s3', 'kyc', rawKey); logObjectAction('SKIPPED (appeared concurrently)', 's3->azure', 'kyc', rawKey); const azureDownloadFailure = { getBlockBlobClient: () => ({ download: jest.fn().mockRejectedValue(new Error(`failed ${rawKey}`)) }), } as never; - await expect(copyAzureToS3(azureDownloadFailure, makeS3Client(), 'kyc', rawKey)).rejects.toThrow(expectedReference); + await expect(copyAzureToS3(azureDownloadFailure, makeS3Client(), 'kyc', rawKey, 42)).rejects.toThrow( + expectedReference, + ); const s3DownloadFailure = { send: jest.fn().mockRejectedValue(new Error(`failed ${rawKey}`)), } as never; - await expect(copyS3ToAzure(s3DownloadFailure, 'kyc', {} as never, rawKey)).rejects.toThrow(expectedReference); + await expect(copyS3ToAzure(s3DownloadFailure, 'kyc', {} as never, rawKey, 42)).rejects.toThrow(expectedReference); const output = consoleSpy.mock.calls.flat().join('\n'); expect(output).toContain(expectedReference); - expect(output).not.toContain(rawKey); - expect(output).not.toContain('private-document'); - expect(RECONCILER_PRIVACY_LOG_VERSION).toBe('storage-reconciler-private-logs-v1'); + expect(output).toContain(`key-sha256=${hashObjectKeySha256(rawKey)}`); + expect(output).toContain('source=azure'); + expect(output).toContain('size=42'); + expect(output).not.toContain('contentType'); + expect(output).not.toContain('content-type'); + assertNoSentinelLeak(output); + assertNoEtagSentinelLeak(output); + expect(RECONCILER_PRIVACY_LOG_VERSION).toBe('storage-reconciler-private-logs-v2'); }); it('redacts raw keys in incomplete S3 and Azure listing errors', async () => { - const rawKey = 'user/456/incomplete-private-document.pdf'; + const rawKey = SENTINEL_KEY_B; const expectedReference = safeObjectReference('kyc', rawKey); s3Mock.reset(); @@ -107,8 +164,7 @@ describe('safeObjectReference', () => { }); const s3Error = await rejectedError(listS3Objects(makeS3Client(), 'kyc')); expect(s3Error.message).toContain(expectedReference); - expect(s3Error.message).not.toContain(rawKey); - expect(s3Error.message).not.toContain('incomplete-private-document'); + assertNoSentinelLeak(s3Error.message); const azureContainer = { containerName: 'kyc', @@ -118,8 +174,40 @@ describe('safeObjectReference', () => { } as never; const azureError = await rejectedError(listAzureObjects(azureContainer)); expect(azureError.message).toContain(expectedReference); - expect(azureError.message).not.toContain(rawKey); - expect(azureError.message).not.toContain('incomplete-private-document'); + assertNoSentinelLeak(azureError.message); + }); +}); + +describe('listing does not capture ETags', () => { + beforeEach(() => { + s3Mock.reset(); + }); + + it('listS3Objects stores only key, size, lastModified (ignores listing ETag)', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ + IsTruncated: false, + Contents: [{ Key: 'a', Size: 10, LastModified: t0, ETag: ETAG_SENTINEL_S3 }], + }); + const objects = await listS3Objects(makeS3Client(), 'kyc'); + expect(objects).toEqual([{ key: 'a', size: 10, lastModified: t0 }]); + expect(objects[0]).not.toHaveProperty('etag'); + assertNoEtagSentinelLeak(JSON.stringify(objects)); + }); + + it('listAzureObjects stores only key, size, lastModified (ignores properties.etag)', async () => { + const azureContainer = { + containerName: 'kyc', + async *listBlobsFlat() { + yield { + name: 'b', + properties: { contentLength: 20, lastModified: t0, etag: ETAG_SENTINEL_AZURE }, + }; + }, + } as never; + const objects = await listAzureObjects(azureContainer); + expect(objects).toEqual([{ key: 'b', size: 20, lastModified: t0 }]); + expect(objects[0]).not.toHaveProperty('etag'); + assertNoEtagSentinelLeak(JSON.stringify(objects)); }); }); @@ -303,6 +391,29 @@ describe('assertWithinHealCap', () => { }); }); +describe('assertExactHealBinding', () => { + const digest = 'a'.repeat(64); + const other = 'b'.repeat(64); + + it('does not throw when count and digest match exactly', () => { + expect(() => assertExactHealBinding(3, 3, digest, digest)).not.toThrow(); + }); + + it('throws on candidate count mismatch before any I/O', () => { + expect(() => assertExactHealBinding(2, 3, digest, digest)).toThrow(/Exact-cap heal binding failed/); + expect(() => assertExactHealBinding(2, 3, digest, digest)).toThrow(/additive candidate count 2/); + expect(() => assertExactHealBinding(2, 3, digest, digest)).toThrow(/RECONCILE_HEAL_CAP 3/); + expect(() => assertExactHealBinding(2, 3, digest, digest)).toThrow(/before any WORM probe, download, or PUT/); + }); + + it('throws on candidate-set digest mismatch before any I/O', () => { + expect(() => assertExactHealBinding(3, 3, digest, other)).toThrow(/Exact-cap heal binding failed/); + expect(() => assertExactHealBinding(3, 3, digest, other)).toThrow(new RegExp(digest)); + expect(() => assertExactHealBinding(3, 3, digest, other)).toThrow(new RegExp(other)); + expect(() => assertExactHealBinding(3, 3, digest, other)).toThrow(/before any WORM probe, download, or PUT/); + }); +}); + describe('assertRequestedContainersExist', () => { it('throws when a requested container is missing from S3 buckets', () => { expect(() => assertRequestedContainersExist(['kyc'], ['kyc', 'support'])).toThrow(/support/); @@ -313,6 +424,187 @@ describe('assertRequestedContainersExist', () => { }); }); +describe('additive summary and candidateSetSha256', () => { + it('is stable and order-independent over the same candidate multiset', () => { + const candidatesForward = [ + { container: 'kyc', direction: 'azureToS3' as const, keySha256: hashObjectKeySha256(SENTINEL_KEY_A), size: 10 }, + { + container: 'support', + direction: 's3ToAzure' as const, + keySha256: hashObjectKeySha256(SENTINEL_KEY_B), + size: 20, + }, + { container: 'kyc', direction: 's3ToAzure' as const, keySha256: hashObjectKeySha256(SENTINEL_KEY_C), size: 30 }, + ]; + const candidatesReversed = [...candidatesForward].reverse(); + const candidatesShuffled = [candidatesForward[1], candidatesForward[2], candidatesForward[0]]; + + const d1 = computeCandidateSetSha256(candidatesForward); + const d2 = computeCandidateSetSha256(candidatesReversed); + const d3 = computeCandidateSetSha256(candidatesShuffled); + + expect(d1).toMatch(/^[a-f0-9]{64}$/); + expect(d1).toBe(d2); + expect(d1).toBe(d3); + assertNoSentinelLeak(d1); + }); + + it('changes when container, direction, key, or size changes', () => { + const base = { + container: 'kyc', + direction: 'azureToS3' as const, + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: 10, + }; + const baseDigest = computeCandidateSetSha256([base]); + + expect(computeCandidateSetSha256([{ ...base, container: 'support' }])).not.toBe(baseDigest); + expect(computeCandidateSetSha256([{ ...base, direction: 's3ToAzure' }])).not.toBe(baseDigest); + expect(computeCandidateSetSha256([{ ...base, keySha256: hashObjectKeySha256(SENTINEL_KEY_B) }])).not.toBe( + baseDigest, + ); + expect(computeCandidateSetSha256([{ ...base, size: 11 }])).not.toBe(baseDigest); + }); + + it('sums candidate bytes per direction and never embeds raw keys in the report JSON', () => { + const azureObjs = [ + storedObject(SENTINEL_KEY_A, 100, t0), + storedObject(SENTINEL_KEY_B, 250, t0), + storedObject('shared', 5, t0), + ]; + const s3Objs = [storedObject(SENTINEL_KEY_C, 40, t0), storedObject('shared', 5, t0)]; + const diff = diffStores(azureObjs, s3Objs, skewMs); + + expect(new Set(diff.onlyOnAzure)).toEqual(new Set([SENTINEL_KEY_A, SENTINEL_KEY_B])); + expect(diff.onlyOnS3).toEqual([SENTINEL_KEY_C]); + + const summary = buildAdditiveReconcileSummary([ + { container: 'kyc', diff, azureObjs, s3Objs }, + { + container: 'support', + diff: { onlyOnAzure: [], onlyOnS3: [], sizeMismatch: [], suspectedOverwrite: [] }, + azureObjs: [], + s3Objs: [], + }, + { + container: 'ep2-example', + diff: { onlyOnAzure: [], onlyOnS3: [], sizeMismatch: [], suspectedOverwrite: [] }, + azureObjs: [], + s3Objs: [], + }, + ]); + + expect(summary.containers).toHaveLength(3); + const kyc = summary.containers.find((c) => c.container === 'kyc'); + expect(kyc?.azureToS3).toEqual({ count: 2, bytes: 350 }); + expect(kyc?.s3ToAzure).toEqual({ count: 1, bytes: 40 }); + expect(summary.totals.azureToS3).toEqual({ count: 2, bytes: 350 }); + expect(summary.totals.s3ToAzure).toEqual({ count: 1, bytes: 40 }); + expect(summary.totals.additiveCandidates).toBe(3); + expect(summary.totals.onlyOnAzure).toBe(2); + expect(summary.totals.onlyOnS3).toBe(1); + expect(summary.candidateSetSha256).toMatch(/^[a-f0-9]{64}$/); + + const azureByKey = indexStoredObjectsByKey(azureObjs); + const s3ByKey = indexStoredObjectsByKey(s3Objs); + const candidates = collectAdditiveCandidatesForContainer('kyc', diff, azureByKey, s3ByKey); + expect(computeCandidateSetSha256(candidates)).toBe(summary.candidateSetSha256); + + const report = buildMachineReadableReconcileReport(summary, 'REPORT', '2024-01-01T00:00:00.000Z', { + apiImageDigest: 'sha256:deadbeef', + operatorCommitSha: 'abc123def', + }); + expect(report.schemaVersion).toBe(RECONCILE_REPORT_SCHEMA_VERSION); + expect(report.privacyLogVersion).toBe('storage-reconciler-private-logs-v2'); + expect(report.mode).toBe('REPORT'); + expect(report.apiImageDigest).toBe('sha256:deadbeef'); + expect(report.operatorCommitSha).toBe('abc123def'); + + const line = formatReconcileReportJsonLine(report); + expect(line.startsWith('RECONCILE_REPORT_JSON=')).toBe(true); + const jsonPart = line.slice('RECONCILE_REPORT_JSON='.length); + const parsed = JSON.parse(jsonPart) as Record; + expect(parsed.candidateSetSha256).toBe(summary.candidateSetSha256); + expect(parsed.schemaVersion).toBe(1); + assertNoSentinelLeak(line); + assertNoSentinelLeak(JSON.stringify(parsed)); + }); +}); + +describe('privacy-safe detail mode', () => { + afterEach(() => jest.restoreAllMocks()); + + it('prints only the technical whitelist and both sides for sizeMismatch (no ETag)', () => { + const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + const azureObj = storedObject(SENTINEL_KEY_A, 100, t0); + const s3Obj = storedObject(SENTINEL_KEY_A, 99, new Date('2024-02-01T00:00:00.000Z')); + const azureByKey = indexStoredObjectsByKey([azureObj]); + const s3ByKey = indexStoredObjectsByKey([s3Obj]); + + printCategory('kyc', 'sizeMismatch', [SENTINEL_KEY_A], true, { + azureByKey, + s3ByKey, + dualSide: true, + }); + + const output = consoleSpy.mock.calls.flat().join('\n'); + expect(output).toContain('sizeMismatch: 1'); + expect(output).toContain(formatDetailObjectLine('kyc', SENTINEL_KEY_A, 'azure', azureObj)); + expect(output).toContain(formatDetailObjectLine('kyc', SENTINEL_KEY_A, 's3', s3Obj)); + expect(output).toContain('source=azure'); + expect(output).toContain('source=s3'); + expect(output).toContain('size=100'); + expect(output).toContain('size=99'); + expect(output).toContain('lastModified=2024-01-01T00:00:00.000Z'); + expect(output).toContain('lastModified=2024-02-01T00:00:00.000Z'); + // Conspicuous ETag sentinels must never appear even if present on listing payloads elsewhere + expect(output).not.toContain(ETAG_SENTINEL_AZURE); + expect(output).not.toContain(ETAG_SENTINEL_S3); + expect(output).not.toMatch(/\betag=/i); + expect(output).not.toContain('contentType'); + expect(output).not.toContain('content-type'); + expect(output).not.toContain('metadata'); + expect(output).not.toContain('user-meta'); + assertNoSentinelLeak(output); + assertNoEtagSentinelLeak(output); + }); + + it('formatDetailObjectLine never emits ETag sentinels or etag= fields', () => { + const obj = storedObject(SENTINEL_KEY_A, 42, t0); + const line = formatDetailObjectLine('kyc', SENTINEL_KEY_A, 'azure', obj); + expect(line).toContain('container=kyc'); + expect(line).toContain(`key-sha256=${hashObjectKeySha256(SENTINEL_KEY_A)}`); + expect(line).toContain('source=azure'); + expect(line).toContain('size=42'); + expect(line).toContain('lastModified=2024-01-01T00:00:00.000Z'); + expect(line).not.toContain(ETAG_SENTINEL_AZURE); + expect(line).not.toContain(ETAG_SENTINEL_S3); + expect(line).not.toMatch(/\betag=/i); + assertNoSentinelLeak(line); + assertNoEtagSentinelLeak(line); + }); + + it('caps verbose samples at 20 entries per category', () => { + const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + const keys: string[] = []; + const objs: StoredObject[] = []; + for (let i = 0; i < 25; i++) { + const key = `key-${i}`; + keys.push(key); + objs.push(storedObject(key, i, t0)); + } + printCategory('kyc', 'onlyOnAzure', keys, true, { + bytes: objs.reduce((s, o) => s + o.size, 0), + azureByKey: indexStoredObjectsByKey(objs), + singleSource: 'azure', + }); + const output = consoleSpy.mock.calls.flat().join('\n'); + expect(output).toContain('... and 5 more'); + const detailLines = consoleSpy.mock.calls.map((c) => String(c[0])).filter((line) => line.includes('key-sha256=')); + expect(detailLines).toHaveLength(20); + }); +}); + describe('parseConfig', () => { const ENV_KEYS = [ 'RECONCILE_CONTAINERS', @@ -320,6 +612,9 @@ describe('parseConfig', () => { 'RECONCILE_WORM_CONTAINERS', 'RECONCILE_HEAL', 'RECONCILE_HEAL_CAP', + 'RECONCILE_EXPECTED_CANDIDATE_SET_SHA256', + 'RECONCILE_API_IMAGE_DIGEST', + 'RECONCILE_OPERATOR_COMMIT_SHA', ] as const; let savedEnv: Record<(typeof ENV_KEYS)[number], string | undefined>; @@ -332,6 +627,9 @@ describe('parseConfig', () => { RECONCILE_WORM_CONTAINERS: process.env.RECONCILE_WORM_CONTAINERS, RECONCILE_HEAL: process.env.RECONCILE_HEAL, RECONCILE_HEAL_CAP: process.env.RECONCILE_HEAL_CAP, + RECONCILE_EXPECTED_CANDIDATE_SET_SHA256: process.env.RECONCILE_EXPECTED_CANDIDATE_SET_SHA256, + RECONCILE_API_IMAGE_DIGEST: process.env.RECONCILE_API_IMAGE_DIGEST, + RECONCILE_OPERATOR_COMMIT_SHA: process.env.RECONCILE_OPERATOR_COMMIT_SHA, }; savedArgv = process.argv; for (const key of ENV_KEYS) { @@ -389,6 +687,7 @@ describe('parseConfig', () => { const cfg = parseConfig(); expect(cfg.heal).toBe(false); expect(cfg.verbose).toBe(false); + expect(cfg.exactCap).toBe(false); }); it('sets heal true from RECONCILE_HEAL=true without --heal flag', () => { @@ -479,6 +778,52 @@ describe('parseConfig', () => { expect(cfg.heal).toBe(false); expect(cfg.wormContainers).toEqual(['kyc', 'typo-bucket']); }); + + it('throws when --exact-cap is used without heal', () => { + process.argv = ['node', 'script', 'kyc', '--exact-cap']; + expect(() => parseConfig()).toThrow(/--exact-cap is only allowed together with --heal/); + }); + + it('throws when --exact-cap lacks RECONCILE_HEAL_CAP (no silent default)', () => { + process.argv = ['node', 'script', 'kyc', '--heal', '--exact-cap']; + process.env.RECONCILE_WORM_CONTAINERS = 'kyc'; + process.env.RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 = 'a'.repeat(64); + expect(() => parseConfig()).toThrow(/RECONCILE_HEAL_CAP is required when --exact-cap/); + }); + + it('throws when --exact-cap lacks RECONCILE_EXPECTED_CANDIDATE_SET_SHA256', () => { + process.argv = ['node', 'script', 'kyc', '--heal', '--exact-cap']; + process.env.RECONCILE_WORM_CONTAINERS = 'kyc'; + process.env.RECONCILE_HEAL_CAP = '2'; + expect(() => parseConfig()).toThrow(/RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 is required/); + }); + + it('throws when expected candidate digest is not 64 lowercase hex', () => { + process.argv = ['node', 'script', 'kyc', '--heal', '--exact-cap']; + process.env.RECONCILE_WORM_CONTAINERS = 'kyc'; + process.env.RECONCILE_HEAL_CAP = '2'; + for (const invalid of ['ABC', 'A'.repeat(64), 'g'.repeat(64), 'a'.repeat(63), 'a'.repeat(65)]) { + process.env.RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 = invalid; + expect(() => parseConfig()).toThrow(/64 lowercase hex/); + } + }); + + it('accepts valid --exact-cap configuration', () => { + const digest = '0123456789abcdef'.repeat(4); + process.argv = ['node', 'script', 'kyc', '--heal', '--exact-cap']; + process.env.RECONCILE_WORM_CONTAINERS = 'kyc'; + process.env.RECONCILE_HEAL_CAP = '7'; + process.env.RECONCILE_EXPECTED_CANDIDATE_SET_SHA256 = digest; + process.env.RECONCILE_API_IMAGE_DIGEST = 'sha256:image'; + process.env.RECONCILE_OPERATOR_COMMIT_SHA = 'deadbeef'; + const cfg = parseConfig(); + expect(cfg.exactCap).toBe(true); + expect(cfg.heal).toBe(true); + expect(cfg.healCap).toBe(7); + expect(cfg.expectedCandidateSetSha256).toBe(digest); + expect(cfg.apiImageDigest).toBe('sha256:image'); + expect(cfg.operatorCommitSha).toBe('deadbeef'); + }); }); describe('assertWormContainersConfig', () => { @@ -706,6 +1051,22 @@ describe('assertBucketWorm / assertBucketWormIfDeclared / assertUndeclaredBucket expect(s3Mock.commandCalls(GetObjectLockConfigurationCommand)).toHaveLength(1); expect(verified.get('support')).toBe(true); }); + + it('WORM probe remains fail-closed before azure→s3 heal path (declared COMPLIANCE failure)', async () => { + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: 'kyc' }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'GOVERNANCE', Years: GEBUEV_RETENTION_FLOOR_YEARS } }, + }, + }); + const client = makeS3Client(); + const verified = new Map(); + const noWormVerified = new Map(); + await expect(assertBucketWormIfDeclared(client, 'kyc', new Set(['kyc']), verified, noWormVerified)).rejects.toThrow( + /Refusing azure→s3 heal/, + ); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); }); describe('assertBucketsAccounted', () => { @@ -779,3 +1140,724 @@ describe('isAzurePreconditionFailed', () => { expect(isAzurePreconditionFailed({ statusCode: 500 })).toBe(false); }); }); + +describe('copyAzureToS3 / copyS3ToAzure preconditions', () => { + beforeEach(() => { + s3Mock.reset(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('S3 PUT includes IfNoneMatch: * and returns healed on success', async () => { + const body = Buffer.from('payload'); + const readable = Readable.from([body]); + const azureContainer = { + getBlockBlobClient: () => ({ + download: jest.fn().mockResolvedValue({ + readableStreamBody: readable, + contentType: 'application/pdf', + metadata: { owner: 'should-not-be-logged' }, + }), + }), + } as never; + + s3Mock.on(PutObjectCommand).resolves({}); + const result = await copyAzureToS3(azureContainer, makeS3Client(), 'kyc', SENTINEL_KEY_A, body.length); + expect(result).toBe('healed'); + + const putCalls = s3Mock.commandCalls(PutObjectCommand); + expect(putCalls).toHaveLength(1); + expect(putCalls[0].args[0].input.IfNoneMatch).toBe('*'); + expect(putCalls[0].args[0].input.Key).toBe(SENTINEL_KEY_A); + }); + + it('S3 precondition conflict is skipped (atomic concurrency)', async () => { + jest.spyOn(console, 'log').mockImplementation(); + const body = Buffer.from('payload'); + const readable = Readable.from([body]); + const azureContainer = { + getBlockBlobClient: () => ({ + download: jest.fn().mockResolvedValue({ + readableStreamBody: readable, + contentType: 'application/octet-stream', + metadata: {}, + }), + }), + } as never; + + s3Mock.on(PutObjectCommand).rejects( + Object.assign(new Error('Precondition Failed'), { + name: 'PreconditionFailed', + $metadata: { httpStatusCode: 412 }, + }), + ); + + const result = await copyAzureToS3(azureContainer, makeS3Client(), 'kyc', SENTINEL_KEY_A, body.length); + expect(result).toBe('skipped'); + }); + + it('Azure upload includes conditions.ifNoneMatch: * and returns healed on success', async () => { + const bodyBytes = new Uint8Array([1, 2, 3]); + s3Mock.on(GetObjectCommand).resolves({ + Body: { + transformToByteArray: async () => bodyBytes, + } as never, + ContentType: 'text/plain', + Metadata: { note: 'should-not-be-logged' }, + }); + + const uploadData = jest.fn().mockResolvedValue(undefined); + const azureContainer = { + getBlockBlobClient: () => ({ uploadData }), + } as never; + + const result = await copyS3ToAzure(makeS3Client(), 'kyc', azureContainer, SENTINEL_KEY_B, bodyBytes.length); + expect(result).toBe('healed'); + expect(uploadData).toHaveBeenCalledTimes(1); + const uploadOpts = uploadData.mock.calls[0][1] as { + conditions: { ifNoneMatch: string }; + metadata?: Record; + }; + expect(uploadOpts.conditions.ifNoneMatch).toBe('*'); + }); + + it('Azure precondition conflict is skipped (atomic concurrency)', async () => { + jest.spyOn(console, 'log').mockImplementation(); + const bodyBytes = new Uint8Array([1, 2, 3]); + s3Mock.on(GetObjectCommand).resolves({ + Body: { + transformToByteArray: async () => bodyBytes, + } as never, + ContentType: 'text/plain', + Metadata: {}, + }); + + const uploadData = jest.fn().mockRejectedValue( + Object.assign(new Error('BlobAlreadyExists'), { + statusCode: 412, + details: { errorCode: 'BlobAlreadyExists' }, + }), + ); + const azureContainer = { + getBlockBlobClient: () => ({ uploadData }), + } as never; + + const result = await copyS3ToAzure(makeS3Client(), 'kyc', azureContainer, SENTINEL_KEY_B, bodyBytes.length); + expect(result).toBe('skipped'); + }); + + it('Azure→S3 rejects when downloaded size differs from inventory size (no PutObject)', async () => { + // Inventory bound 3 bytes; source grew to 4 between list and download. + const inventorySize = 3; + const downloaded = Buffer.from([1, 2, 3, 4]); + const azureContainer = { + getBlockBlobClient: () => ({ + download: jest.fn().mockResolvedValue({ + readableStreamBody: Readable.from([downloaded]), + contentType: 'application/octet-stream', + metadata: { owner: 'should-not-be-logged' }, + }), + }), + } as never; + + const err = await rejectedError( + copyAzureToS3(azureContainer, makeS3Client(), 'kyc', SENTINEL_KEY_A, inventorySize), + ); + + expect(err.message).toMatch(/Source size race for azure->s3/); + expect(err.message).toContain('expected 3 bytes'); + expect(err.message).toContain('got 4 bytes'); + expect(err.message).toContain(safeObjectReference('kyc', SENTINEL_KEY_A)); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('S3→Azure rejects when downloaded size differs from inventory size (no uploadData)', async () => { + // Inventory bound 3 bytes; source grew to 4 between list and download. + const inventorySize = 3; + const downloaded = new Uint8Array([1, 2, 3, 4]); + s3Mock.on(GetObjectCommand).resolves({ + Body: { + transformToByteArray: async () => downloaded, + } as never, + ContentType: 'text/plain', + Metadata: { note: 'should-not-be-logged' }, + }); + + const uploadData = jest.fn().mockResolvedValue(undefined); + const azureContainer = { + getBlockBlobClient: () => ({ uploadData }), + } as never; + + const err = await rejectedError( + copyS3ToAzure(makeS3Client(), 'kyc', azureContainer, SENTINEL_KEY_B, inventorySize), + ); + + expect(err.message).toMatch(/Source size race for s3->azure/); + expect(err.message).toContain('expected 3 bytes'); + expect(err.message).toContain('got 4 bytes'); + expect(err.message).toContain(safeObjectReference('kyc', SENTINEL_KEY_B)); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + expect(uploadData).not.toHaveBeenCalled(); + }); + + it('Azure→S3 rejects invalid expectedSourceSize before download or PutObject', async () => { + const download = jest.fn(); + const azureContainer = { + getBlockBlobClient: () => ({ download }), + } as never; + + for (const invalid of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1]) { + const err = await rejectedError(copyAzureToS3(azureContainer, makeS3Client(), 'kyc', SENTINEL_KEY_A, invalid)); + expect(err.message).toMatch(/Invalid expectedSourceSize for azure->s3/); + expect(err.message).toContain(safeObjectReference('kyc', SENTINEL_KEY_A)); + assertNoSentinelLeak(err.message); + } + + expect(download).not.toHaveBeenCalled(); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('S3→Azure rejects invalid expectedSourceSize before GetObject or uploadData', async () => { + const uploadData = jest.fn(); + const azureContainer = { + getBlockBlobClient: () => ({ uploadData }), + } as never; + + for (const invalid of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1]) { + const err = await rejectedError(copyS3ToAzure(makeS3Client(), 'kyc', azureContainer, SENTINEL_KEY_B, invalid)); + expect(err.message).toMatch(/Invalid expectedSourceSize for s3->azure/); + expect(err.message).toContain(safeObjectReference('kyc', SENTINEL_KEY_B)); + assertNoSentinelLeak(err.message); + } + + expect(uploadData).not.toHaveBeenCalled(); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + }); +}); + +describe('runAdditiveHealOrchestration (production HEAL path)', () => { + beforeEach(() => { + s3Mock.reset(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + function makeAzureClient(opts?: { download?: jest.Mock; uploadData?: jest.Mock; getContainerClient?: jest.Mock }): { + getContainerClient: jest.Mock; + download: jest.Mock; + uploadData: jest.Mock; + } { + const download = opts?.download ?? jest.fn(); + const uploadData = opts?.uploadData ?? jest.fn(); + const getContainerClient = + opts?.getContainerClient ?? + jest.fn().mockReturnValue({ + getBlockBlobClient: () => ({ download, uploadData }), + }); + return { getContainerClient, download, uploadData }; + } + + function emptyHealReport(container = 'kyc'): HealContainerReport { + return { + container, + diff: { + onlyOnAzure: [] as string[], + onlyOnS3: [] as string[], + sizeMismatch: [] as string[], + suspectedOverwrite: [] as string[], + }, + azureByKey: new Map(), + s3ByKey: new Map(), + }; + } + + function azureOnlyHealReport(container: string, key: string, size: number): HealContainerReport { + const obj = storedObject(key, size, t0); + return { + container, + diff: { + onlyOnAzure: [key], + onlyOnS3: [] as string[], + sizeMismatch: [] as string[], + suspectedOverwrite: [] as string[], + }, + azureByKey: indexStoredObjectsByKey([obj]), + s3ByKey: new Map(), + }; + } + + function assertZeroStorageAndWormIo(): void { + expect(s3Mock.commandCalls(GetObjectLockConfigurationCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + } + + it('rejects exact count mismatch with zero storage/WORM I/O', async () => { + const azure = makeAzureClient(); + const report = azureOnlyHealReport('kyc', SENTINEL_KEY_A, 10); + const actualDigest = computeCandidateSetSha256([ + { + container: 'kyc', + direction: 'azureToS3', + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: 10, + }, + ]); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [report], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 2, + exactCap: true, + actualCandidateSetSha256: actualDigest, + expectedCandidateSetSha256: actualDigest, + }), + ); + + expect(err.message).toMatch(/Exact-cap heal binding failed/); + expect(err.message).toMatch(/additive candidate count 1/); + expect(err.message).toMatch(/RECONCILE_HEAL_CAP 2/); + expect(err.message).toMatch(/before any WORM probe, download, or PUT/); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + expect(azure.getContainerClient).not.toHaveBeenCalled(); + expect(azure.download).not.toHaveBeenCalled(); + expect(azure.uploadData).not.toHaveBeenCalled(); + assertZeroStorageAndWormIo(); + }); + + it('rejects same-count digest mismatch with zero storage/WORM I/O', async () => { + const azure = makeAzureClient(); + const report = azureOnlyHealReport('kyc', SENTINEL_KEY_A, 10); + const actualDigest = computeCandidateSetSha256([ + { + container: 'kyc', + direction: 'azureToS3', + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: 10, + }, + ]); + const expectedDigest = 'b'.repeat(64); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [report], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 1, + exactCap: true, + actualCandidateSetSha256: actualDigest, + expectedCandidateSetSha256: expectedDigest, + }), + ); + + expect(err.message).toMatch(/Exact-cap heal binding failed/); + expect(err.message).toContain(actualDigest); + expect(err.message).toContain(expectedDigest); + expect(err.message).toMatch(/before any WORM probe, download, or PUT/); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + expect(azure.getContainerClient).not.toHaveBeenCalled(); + expect(azure.download).not.toHaveBeenCalled(); + expect(azure.uploadData).not.toHaveBeenCalled(); + assertZeroStorageAndWormIo(); + }); + + it('rejects empty live candidate set under positive exact-cap (no clean-parity bypass)', async () => { + const azure = makeAzureClient(); + const emptyDigest = computeCandidateSetSha256([]); + // Approved positive cap/digest from a prior report, but live inventory is now empty. + const staleApprovedDigest = 'c'.repeat(64); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [emptyHealReport('kyc')], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 3, + exactCap: true, + actualCandidateSetSha256: emptyDigest, + expectedCandidateSetSha256: staleApprovedDigest, + }), + ); + + expect(err.message).toMatch(/Exact-cap heal binding failed/); + expect(err.message).toMatch(/additive candidate count 0/); + expect(err.message).toMatch(/RECONCILE_HEAL_CAP 3/); + expect(err.message).toMatch(/before any WORM probe, download, or PUT/); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + expect(azure.getContainerClient).not.toHaveBeenCalled(); + expect(azure.download).not.toHaveBeenCalled(); + expect(azure.uploadData).not.toHaveBeenCalled(); + assertZeroStorageAndWormIo(); + }); + + it('rejects WORM failure for azure→s3 after probe but before Azure download and S3 PutObject', async () => { + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: 'kyc' }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'GOVERNANCE', Years: GEBUEV_RETENTION_FLOOR_YEARS } }, + }, + }); + + const azure = makeAzureClient({ + download: jest.fn().mockResolvedValue({ + readableStreamBody: Readable.from([Buffer.from('payload')]), + contentType: 'application/octet-stream', + metadata: {}, + }), + }); + const report = azureOnlyHealReport('kyc', SENTINEL_KEY_A, 7); + const actualDigest = computeCandidateSetSha256([ + { + container: 'kyc', + direction: 'azureToS3', + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: 7, + }, + ]); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [report], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 1, + exactCap: false, + actualCandidateSetSha256: actualDigest, + }), + ); + + expect(err.message).toMatch(/Refusing azure→s3 heal/); + assertNoSentinelLeak(err.message); + assertNoEtagSentinelLeak(err.message); + // WORM probe is expected; copy I/O must not follow. + expect(s3Mock.commandCalls(GetObjectLockConfigurationCommand)).toHaveLength(1); + expect(azure.getContainerClient).toHaveBeenCalledWith('kyc'); + expect(azure.download).not.toHaveBeenCalled(); + expect(azure.uploadData).not.toHaveBeenCalled(); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('normal non-exact HEAL with zero candidates is a no-op success after max-cap auth', async () => { + const azure = makeAzureClient(); + const stats = await runAdditiveHealOrchestration({ + reports: [emptyHealReport('kyc')], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: DEFAULT_HEAL_CAP, + exactCap: false, + actualCandidateSetSha256: computeCandidateSetSha256([]), + }); + + expect(stats.azureToS3).toEqual({ + healedCount: 0, + healedBytes: 0, + skippedCount: 0, + skippedBytes: 0, + }); + expect(stats.s3ToAzure).toEqual({ + healedCount: 0, + healedBytes: 0, + skippedCount: 0, + skippedBytes: 0, + }); + // Container client may be resolved, but no WORM probe / download / PUT / upload. + expect(azure.download).not.toHaveBeenCalled(); + expect(azure.uploadData).not.toHaveBeenCalled(); + assertZeroStorageAndWormIo(); + }); + + it('exact-cap success heals both directions with WORM-before-copy ordering, conditional targets, stats, and privacy', async () => { + const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + const callOrder: string[] = []; + + // Inventory sizes must equal the buffers that will be downloaded (digest-bound (key,size) gate). + const azurePayload = Buffer.from('azure-source-payload'); + const s3Payload = new Uint8Array([9, 8, 7, 6]); + const azureToS3Size = azurePayload.length; + const s3ToAzureSize = s3Payload.length; + const contentTypeSentinel = 'application/pdf-NEVER-LOG-CONTENT-TYPE'; + const reverseContentTypeSentinel = 'text/plain-NEVER-LOG-CONTENT-TYPE'; + const userMetaSentinel = 'user-meta-NEVER-LOG-OWNER'; + + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: 'kyc' }).callsFake(async () => { + callOrder.push('worm-probe'); + return { + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: GEBUEV_RETENTION_FLOOR_YEARS } }, + }, + }; + }); + + s3Mock.on(PutObjectCommand).callsFake(async () => { + callOrder.push('s3-put'); + return {}; + }); + + s3Mock.on(GetObjectCommand).callsFake(async () => { + callOrder.push('s3-get'); + return { + Body: { + transformToByteArray: async () => s3Payload, + } as never, + ContentType: reverseContentTypeSentinel, + Metadata: { note: userMetaSentinel }, + }; + }); + + const download = jest.fn().mockImplementation(async () => { + callOrder.push('azure-download'); + return { + readableStreamBody: Readable.from([azurePayload]), + contentType: contentTypeSentinel, + metadata: { owner: userMetaSentinel }, + }; + }); + + const uploadData = jest.fn().mockImplementation(async () => { + callOrder.push('azure-upload'); + }); + + const getBlockBlobClientKeys: string[] = []; + const getContainerClient = jest.fn().mockImplementation((container: string) => ({ + getBlockBlobClient: (key: string) => { + getBlockBlobClientKeys.push(`${container}:${key}`); + return { download, uploadData }; + }, + })); + + const azure = makeAzureClient({ download, uploadData, getContainerClient }); + + const wormAzureToS3Report: HealContainerReport = { + container: 'kyc', + diff: { + onlyOnAzure: [SENTINEL_KEY_A], + onlyOnS3: [], + sizeMismatch: [], + suspectedOverwrite: [], + }, + azureByKey: indexStoredObjectsByKey([storedObject(SENTINEL_KEY_A, azureToS3Size, t0)]), + s3ByKey: new Map(), + }; + + const normalS3ToAzureReport: HealContainerReport = { + container: 'support', + diff: { + onlyOnAzure: [], + onlyOnS3: [SENTINEL_KEY_B], + sizeMismatch: [], + suspectedOverwrite: [], + }, + azureByKey: new Map(), + s3ByKey: indexStoredObjectsByKey([storedObject(SENTINEL_KEY_B, s3ToAzureSize, t0)]), + }; + + const expectedDigest = computeCandidateSetSha256([ + { + container: 'kyc', + direction: 'azureToS3', + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: azureToS3Size, + }, + { + container: 'support', + direction: 's3ToAzure', + keySha256: hashObjectKeySha256(SENTINEL_KEY_B), + size: s3ToAzureSize, + }, + ]); + + const stats = await runAdditiveHealOrchestration({ + reports: [wormAzureToS3Report, normalS3ToAzureReport], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 2, + exactCap: true, + actualCandidateSetSha256: expectedDigest, + expectedCandidateSetSha256: expectedDigest, + }); + + // Real production sequence: WORM probe before Azure download + S3 PUT; reverse is S3 GET + Azure upload. + expect(callOrder).toEqual(['worm-probe', 'azure-download', 's3-put', 's3-get', 'azure-upload']); + + const wormCalls = s3Mock.commandCalls(GetObjectLockConfigurationCommand); + expect(wormCalls).toHaveLength(1); + expect(wormCalls[0].args[0].input.Bucket).toBe('kyc'); + + const putCalls = s3Mock.commandCalls(PutObjectCommand); + expect(putCalls).toHaveLength(1); + expect(putCalls[0].args[0].input.Bucket).toBe('kyc'); + expect(putCalls[0].args[0].input.Key).toBe(SENTINEL_KEY_A); + expect(putCalls[0].args[0].input.IfNoneMatch).toBe('*'); + // Proves orchestration passed the digest-bound inventory size into azure→s3 copy: + // only matching expectedSourceSize allows PutObject with this Body length. + const putBody = putCalls[0].args[0].input.Body as Buffer; + expect(Buffer.isBuffer(putBody) ? putBody.length : 0).toBe(azureToS3Size); + + const getCalls = s3Mock.commandCalls(GetObjectCommand); + expect(getCalls).toHaveLength(1); + expect(getCalls[0].args[0].input.Bucket).toBe('support'); + expect(getCalls[0].args[0].input.Key).toBe(SENTINEL_KEY_B); + + expect(uploadData).toHaveBeenCalledTimes(1); + const uploadBody = uploadData.mock.calls[0][0] as Buffer; + // Proves orchestration passed the digest-bound inventory size into s3→azure copy. + expect(Buffer.isBuffer(uploadBody) ? uploadBody.length : 0).toBe(s3ToAzureSize); + const uploadOpts = uploadData.mock.calls[0][1] as { + conditions: { ifNoneMatch: string }; + metadata?: Record; + }; + expect(uploadOpts.conditions.ifNoneMatch).toBe('*'); + + expect(getBlockBlobClientKeys).toEqual([`kyc:${SENTINEL_KEY_A}`, `support:${SENTINEL_KEY_B}`]); + expect(azure.getContainerClient).toHaveBeenCalledWith('kyc'); + expect(azure.getContainerClient).toHaveBeenCalledWith('support'); + + expect(stats.azureToS3).toEqual({ + healedCount: 1, + healedBytes: azureToS3Size, + skippedCount: 0, + skippedBytes: 0, + }); + expect(stats.s3ToAzure).toEqual({ + healedCount: 1, + healedBytes: s3ToAzureSize, + skippedCount: 0, + skippedBytes: 0, + }); + + const output = consoleSpy.mock.calls.flat().join('\n'); + const azureToS3Ref = safeObjectReference('kyc', SENTINEL_KEY_A); + const s3ToAzureRef = safeObjectReference('support', SENTINEL_KEY_B); + expect(output).toContain(`HEALED azure->s3 ${azureToS3Ref}`); + expect(output).toContain(`HEALED s3->azure ${s3ToAzureRef}`); + assertNoSentinelLeak(output); + assertNoEtagSentinelLeak(output); + expect(output).not.toContain(contentTypeSentinel); + expect(output).not.toContain(reverseContentTypeSentinel); + expect(output).not.toContain(userMetaSentinel); + expect(output).not.toContain('contentType'); + expect(output).not.toContain('content-type'); + expect(output).not.toContain('metadata'); + expect(output).not.toContain('user-meta'); + }); + + it('orchestration azure→s3 fails closed when download size differs from inventory size', async () => { + // Inventory bound size=3; download returns 4 bytes — proves size is passed into copyAzureToS3. + const inventorySize = 3; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: 'kyc' }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: GEBUEV_RETENTION_FLOOR_YEARS } }, + }, + }); + + const download = jest.fn().mockResolvedValue({ + readableStreamBody: Readable.from([Buffer.from([1, 2, 3, 4])]), + contentType: 'application/octet-stream', + metadata: {}, + }); + const uploadData = jest.fn(); + const azure = makeAzureClient({ download, uploadData }); + const report = azureOnlyHealReport('kyc', SENTINEL_KEY_A, inventorySize); + const actualDigest = computeCandidateSetSha256([ + { + container: 'kyc', + direction: 'azureToS3', + keySha256: hashObjectKeySha256(SENTINEL_KEY_A), + size: inventorySize, + }, + ]); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [report], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 1, + exactCap: false, + actualCandidateSetSha256: actualDigest, + }), + ); + + expect(err.message).toMatch(/Source size race for azure->s3/); + expect(err.message).toContain('expected 3 bytes'); + expect(err.message).toContain('got 4 bytes'); + assertNoSentinelLeak(err.message); + expect(download).toHaveBeenCalled(); + expect(uploadData).not.toHaveBeenCalled(); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('orchestration s3→azure fails closed when download size differs from inventory size', async () => { + // Inventory bound size=3; download returns 4 bytes — proves size is passed into copyS3ToAzure. + const inventorySize = 3; + s3Mock.on(GetObjectCommand).resolves({ + Body: { + transformToByteArray: async () => new Uint8Array([1, 2, 3, 4]), + } as never, + ContentType: 'text/plain', + Metadata: {}, + }); + + const uploadData = jest.fn(); + const azure = makeAzureClient({ uploadData }); + const report: HealContainerReport = { + container: 'support', + diff: { + onlyOnAzure: [], + onlyOnS3: [SENTINEL_KEY_B], + sizeMismatch: [], + suspectedOverwrite: [], + }, + azureByKey: new Map(), + s3ByKey: indexStoredObjectsByKey([storedObject(SENTINEL_KEY_B, inventorySize, t0)]), + }; + const actualDigest = computeCandidateSetSha256([ + { + container: 'support', + direction: 's3ToAzure', + keySha256: hashObjectKeySha256(SENTINEL_KEY_B), + size: inventorySize, + }, + ]); + + const err = await rejectedError( + runAdditiveHealOrchestration({ + reports: [report], + s3: makeS3Client(), + azure, + wormContainers: new Set(['kyc']), + healCap: 1, + exactCap: false, + actualCandidateSetSha256: actualDigest, + }), + ); + + expect(err.message).toMatch(/Source size race for s3->azure/); + expect(err.message).toContain('expected 3 bytes'); + expect(err.message).toContain('got 4 bytes'); + assertNoSentinelLeak(err.message); + expect(uploadData).not.toHaveBeenCalled(); + }); +});