From 247d2d32969dd0deaea649d1b12c03330b361a78 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:28:45 +0200 Subject: [PATCH 01/31] feat(lab): CL-02 immutable evidence ledger and SQLite projection Persist Compatibility Lab evidence as append-only JSONL with content-addressed artifacts and a disposable rebuildable SQLite projection, including CL-01 observation persistence without starting CL-03 probe work. --- .../001_pr_stack_status.md | 33 +- src/lab/artifacts/sanitize.ts | 106 +++ src/lab/artifacts/secure-fs.ts | 374 ++++++++++ src/lab/artifacts/store.ts | 240 +++++++ src/lab/constants.ts | 107 +++ src/lab/digest.ts | 64 ++ src/lab/events/types.ts | 228 ++++++ src/lab/events/validate.ts | 676 ++++++++++++++++++ src/lab/index.ts | 15 + src/lab/ledger/invalidation.ts | 136 ++++ src/lab/ledger/purge.ts | 97 +++ src/lab/ledger/store.ts | 146 ++++ src/lab/observe/from-conformance.ts | 258 +++++++ src/lab/paths.ts | 39 + src/lab/projection/rebuild.ts | 378 ++++++++++ src/lab/projection/schema.ts | 122 ++++ src/lab/projection/verdicts.ts | 301 ++++++++ tests/lab-evidence-ledger.test.ts | 632 ++++++++++++++++ 18 files changed, 3949 insertions(+), 3 deletions(-) create mode 100644 src/lab/artifacts/sanitize.ts create mode 100644 src/lab/artifacts/secure-fs.ts create mode 100644 src/lab/artifacts/store.ts create mode 100644 src/lab/constants.ts create mode 100644 src/lab/digest.ts create mode 100644 src/lab/events/types.ts create mode 100644 src/lab/events/validate.ts create mode 100644 src/lab/index.ts create mode 100644 src/lab/ledger/invalidation.ts create mode 100644 src/lab/ledger/purge.ts create mode 100644 src/lab/ledger/store.ts create mode 100644 src/lab/observe/from-conformance.ts create mode 100644 src/lab/paths.ts create mode 100644 src/lab/projection/rebuild.ts create mode 100644 src/lab/projection/schema.ts create mode 100644 src/lab/projection/verdicts.ts create mode 100644 tests/lab-evidence-ledger.test.ts diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 678e023c2..906435001 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -20,11 +20,17 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | *(this branch tip)* | *(draft upstream PR)* | IMPLEMENTATION IN REVIEW | +| CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. +CL-02 starts from the exact CL-01 merge commit on `dev` +(`4bb249b756abd468c675d2d92fffe4da95ad3e2a` / upstream #1320). It does **not** +base on the pre-merge CL-01 feature branch tip. + ## CL-00 acceptance log - Live-tree audit covered provider registry/derivation, Routing Profiles, @@ -62,6 +68,7 @@ Independent CL-00 acceptance review is frozen at - **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` - **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) - **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `22d608c82d82e2746c0cef9cd761db19a8e465ee` +- **Merged to `dev`:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` via upstream [#1320](https://github.com/lidge-jun/opencodex/pull/1320). ### Corrections applied @@ -88,8 +95,28 @@ Independent CL-00 acceptance review is frozen at - None for CL-01 correction. - Full-suite green remains unavailable on this host for documented Windows/Bun reasons. +## CL-02 implementation log + +- **Branch:** `feat/cl-02-evidence-ledger` +- **Starting/base SHA:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` (CL-01 merge via #1320) +- **Scope:** immutable JSONL evidence ledger, content-addressed artifact store, + disposable/rebuildable SQLite projection, ClaimSourceManifestV1, invalidation + and sensitive purge tombstones, CL-01 → observation persistence seam. +- **Explicitly out of scope:** CL-03 live probes, CL-04 CLI/API, CL-05 UI, + CL-06 profile fields, Fabric, shadow workflows. + +### Boundary note (verdict algorithm) + +CL-02 implements reusable projection primitives and a conservative +protocol-conformance projection sufficient to persist and rebuild derived +verdict rows with required provenance (`asOf`, projection spec version, +manifest digests, contributing event IDs). Full suite-manifest-driven coverage +rules for live-route and task layers remain later-phase work; claims cannot +produce `PROBED`/`VERIFIED`. + ## Authorization - CL-00: **ACCEPTED** (merged #1286). -- CL-01: **ACCEPTED (contract-corrected revalidation)** — ready for stack review against `dev`. -- CL-02: **NOT STARTED / NOT AUTHORIZED**. +- CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. +- CL-02: **IMPLEMENTATION IN REVIEW** on `feat/cl-02-evidence-ledger`. +- CL-03: **NOT STARTED**. diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts new file mode 100644 index 000000000..03bd73962 --- /dev/null +++ b/src/lab/artifacts/sanitize.ts @@ -0,0 +1,106 @@ +/** + * Deny-by-default sanitization before artifact hashing/writing. + * Never persists prompts, secrets, paths, account IDs, raw URLs, or provider bodies. + */ +import type { ArtifactClass } from "../constants"; +import { MAX_SANITIZED_STRING_FIELD } from "../constants"; +import { jcsStringify } from "../digest"; +import { redactSecretString } from "../../lib/redact"; + +const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; +const SECRETISH = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; + +export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { + if ( + artifactClass === "fixture" || + artifactClass === "scenario_manifest" || + artifactClass === "suite_manifest" || + artifactClass === "claim_source_manifest" + ) { + // Contract artifacts are already synthetic/canonical. Mutating them would + // invalidate content-addressed digests; reject secret-shaped material instead. + assertNoSecretMaterial(payload, 0); + return payload; + } + return scrubValue(payload, 0); +} + +const FORBIDDEN_CONTRACT_KEYS = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|baseUrl|hostname|account|alias)$/i; + +function assertNoSecretMaterial(value: unknown, depth: number): void { + if (depth > 8) return; + if (typeof value === "string") { + if (SECRETISH.test(value)) { + throw new Error("contract artifact contains forbidden secret-shaped material"); + } + return; + } + if (Array.isArray(value)) { + for (const item of value) assertNoSecretMaterial(item, depth + 1); + return; + } + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value as object)) { + if (FORBIDDEN_CONTRACT_KEYS.test(key)) { + throw new Error(`contract artifact forbids key ${key}`); + } + assertNoSecretMaterial(child, depth + 1); + } + } +} + +function scrubValue(value: unknown, depth: number): unknown { + if (depth > 8) return "[truncated_depth]"; + if (value === null || typeof value === "boolean" || typeof value === "number") return value; + if (typeof value === "string") return scrubString(value); + if (value instanceof Uint8Array) { + const text = new TextDecoder().decode(value); + return new TextEncoder().encode(scrubString(text)); + } + if (Array.isArray(value)) { + if (value.length > 256) return value.slice(0, 256).map((v) => scrubValue(v, depth + 1)); + return value.map((v) => scrubValue(v, depth + 1)); + } + if (typeof value === "object") { + const out: Record = {}; + const keys = Object.keys(value as object).slice(0, 64); + for (const key of keys) { + if (FORBIDDEN_KEY.test(key)) { + out[key] = "[redacted]"; + continue; + } + out[key] = scrubValue((value as Record)[key], depth + 1); + } + return out; + } + return "[unsupported]"; +} + +function scrubString(value: string): string { + let s = redactSecretString(value); + if (SECRETISH.test(s)) s = s.replace(SECRETISH, "[REDACTED]"); + // Strip absolute filesystem paths (coarse) + s = s.replace(/(?:[A-Za-z]:\\|\/(?:home|Users|tmp|var|etc)\/)[^\s"']+/g, "[path]"); + // Strip URL userinfo / private hosts roughly + s = s.replace(/https?:\/\/[^\s"']+/gi, (url) => { + try { + const u = new URL(url); + if (u.username || u.password) return "[redacted-url]"; + if (/^(localhost|127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[0-1])\.)/i.test(u.hostname)) { + return `${u.protocol}//[private-host]${u.pathname}`; + } + return `${u.protocol}//[host]${u.pathname}`; + } catch { + return "[redacted-url]"; + } + }); + const bytes = new TextEncoder().encode(s); + if (bytes.byteLength > MAX_SANITIZED_STRING_FIELD) { + return new TextDecoder().decode(bytes.slice(0, MAX_SANITIZED_STRING_FIELD)); + } + return s; +} + +export function sanitizedJsonBytes(value: unknown): Uint8Array { + return new TextEncoder().encode(jcsStringify(scrubValue(value, 0))); +} diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts new file mode 100644 index 000000000..c5ddd7b8c --- /dev/null +++ b/src/lab/artifacts/secure-fs.ts @@ -0,0 +1,374 @@ +/** + * Descriptor/handle-bound, no-follow artifact I/O for the Compatibility Lab store. + * + * Pathname-only exists→stat→readFile flows are rejected by contract (040). + * Where the platform cannot enforce equivalent guarantees, operations fail + * closed as harness_failure. + */ +import { + closeSync, + constants as fsConstants, + existsSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + renameSync, + unlinkSync, + writeSync, + type Stats, +} from "node:fs"; +import { join, resolve, sep } from "node:path"; +import { + ARTIFACT_FILENAME_EXT, + MAX_BYTES_PER_ARTIFACT, +} from "../constants"; +import { artifactBytesDigest, isSha256Hex } from "../digest"; + +export class ArtifactFsError extends Error { + readonly code: string; + constructor(code: string, message: string) { + super(message); + this.name = "ArtifactFsError"; + this.code = code; + } +} + +export function harnessFailure(message: string): never { + throw new ArtifactFsError("harness_failure", message); +} + +const O_RDONLY = fsConstants.O_RDONLY; +const O_RDWR = fsConstants.O_RDWR; +const O_CREAT = fsConstants.O_CREAT; +const O_EXCL = fsConstants.O_EXCL; +const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; +const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; + +export function assertDigestName(digest: string): string { + if (!isSha256Hex(digest)) harnessFailure("artifact digest must be lowercase sha256 hex"); + if (digest.includes("/") || digest.includes("\\") || digest.includes(":") || digest.includes("..")) { + harnessFailure("artifact digest must not contain path separators"); + } + return digest; +} + +export function digestFileName(digest: string): string { + return `${assertDigestName(digest)}${ARTIFACT_FILENAME_EXT}`; +} + +function platformSupportsNoFollow(): boolean { + return typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0; +} + +function openFlags(base: number, noFollow: boolean): number { + if (noFollow) { + if (!platformSupportsNoFollow()) { + // Windows: O_NOFOLLOW unavailable — enforce via lstat/fstat/realpath identity checks. + return base; + } + return base | O_NOFOLLOW!; + } + return base; +} + +function assertRegularFileStats(stats: Stats, label: string): void { + if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { + harnessFailure(`${label}: not a regular file`); + } + if (stats.nlink !== 1) { + harnessFailure(`${label}: hard links prohibited (nlink=${stats.nlink})`); + } +} + +function assertDirectoryStats(stats: Stats, label: string): void { + if (!stats.isDirectory() || stats.isSymbolicLink()) { + harnessFailure(`${label}: must be a real directory (no symlink/reparse redirection)`); + } +} + +export interface TrustedArtifactDir { + path: string; + realPath: string; + /** Platform identity token captured at open (dev:ino or equivalent). */ + identity: string; +} + +function identityOf(stats: Stats): string { + return `${stats.dev}:${stats.ino}`; +} + +/** Open and pin a trusted artifacts directory handle identity. */ +export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir { + const abs = resolve(artifactsDir); + if (abs.includes("\0")) harnessFailure("NUL in artifacts path"); + mkdirSync(abs, { recursive: true, mode: 0o700 }); + + let dirFd: number | null = null; + try { + if (typeof O_DIRECTORY === "number" && platformSupportsNoFollow()) { + dirFd = openSync(abs, openFlags(O_RDONLY | O_DIRECTORY, true)); + const stats = fstatSync(dirFd); + assertDirectoryStats(stats, "artifacts dir"); + const realPath = realpathSync.native?.(abs) ?? realpathSync(abs); + if (resolve(realPath) !== resolve(abs) && !realPath.startsWith(abs) && abs !== realPath) { + // Allow only when realpath equals the created path (no redirect). + } + const realResolved = resolve(realPath); + // Reject if realpath escapes or is a different object via symlink redirect. + if (realResolved !== resolve(abs)) { + // On some platforms mkdir creates path that realpath normalizes (drive letter). Compare carefully. + const a = resolve(abs).replace(/\\/g, "/").toLowerCase(); + const b = realResolved.replace(/\\/g, "/").toLowerCase(); + if (a !== b) harnessFailure("artifacts directory realpath mismatch (possible reparse redirect)"); + } + return { path: abs, realPath: realResolved, identity: identityOf(stats) }; + } + + // Windows / platforms without O_DIRECTORY|O_NOFOLLOW: fail-closed checks via lstat+realpath. + const stats = lstatSync(abs); + assertDirectoryStats(stats, "artifacts dir"); + const realPath = resolve(realpathSync.native?.(abs) ?? realpathSync(abs)); + const absNorm = resolve(abs).replace(/\\/g, "/").toLowerCase(); + const realNorm = realPath.replace(/\\/g, "/").toLowerCase(); + if (absNorm !== realNorm) { + harnessFailure("artifacts directory realpath mismatch (possible reparse redirect)"); + } + return { path: abs, realPath, identity: identityOf(stats) }; + } finally { + if (dirFd !== null) closeSync(dirFd); + } +} + +function revalidateDir(dir: TrustedArtifactDir): void { + const stats = lstatSync(dir.path); + assertDirectoryStats(stats, "artifacts dir"); + if (identityOf(stats) !== dir.identity) { + harnessFailure("artifacts directory identity changed"); + } + const realPath = resolve(realpathSync.native?.(dir.path) ?? realpathSync(dir.path)); + if (realPath.replace(/\\/g, "/").toLowerCase() !== dir.realPath.replace(/\\/g, "/").toLowerCase()) { + harnessFailure("artifacts directory realpath changed"); + } +} + +function artifactAbsPath(dir: TrustedArtifactDir, digest: string): string { + const name = digestFileName(digest); + const full = join(dir.path, name); + const resolved = resolve(full); + const root = resolve(dir.path); + if (!resolved.startsWith(root + sep) && resolved !== root) { + // Windows drive-letter case + const r = resolved.replace(/\\/g, "/").toLowerCase(); + const base = root.replace(/\\/g, "/").toLowerCase(); + if (!r.startsWith(base + "/") && r !== base) { + harnessFailure("artifact path escaped artifacts directory"); + } + } + return resolved; +} + +export interface StoredArtifactBytes { + digest: string; + bytes: Uint8Array; + byteCount: number; +} + +/** Write already-redacted bytes under content-addressed name; dedupe on verified match. */ +export function putArtifactBytes( + dir: TrustedArtifactDir, + bytes: Uint8Array, + expectedDigest?: string, +): StoredArtifactBytes { + revalidateDir(dir); + if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) { + harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + const digest = artifactBytesDigest(bytes); + if (expectedDigest !== undefined) { + assertDigestName(expectedDigest); + if (digest !== expectedDigest) harnessFailure("artifact digest mismatch before write"); + } + const target = artifactAbsPath(dir, digest); + + // Reuse existing object only after same-descriptor verification. + if (existsSync(target)) { + const existing = readArtifactBytes(dir, digest, bytes.byteLength); + if (existing.digest !== digest || existing.byteCount !== bytes.byteLength) { + harnessFailure("existing artifact failed verification"); + } + // Constant-time-ish compare + if (existing.bytes.byteLength !== bytes.byteLength) harnessFailure("existing artifact size mismatch"); + let diff = 0; + for (let i = 0; i < bytes.byteLength; i++) diff |= existing.bytes[i]! ^ bytes[i]!; + if (diff !== 0) harnessFailure("existing artifact content mismatch"); + return existing; + } + + const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; + if (tmpName.includes("..") || tmpName.includes("/") || tmpName.includes("\\")) { + harnessFailure("invalid temp name"); + } + const tmpPath = join(dir.path, tmpName); + let fd: number | null = null; + try { + fd = openSync(tmpPath, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); + const written = writeSync(fd, bytes); + if (written !== bytes.byteLength) harnessFailure("short write"); + fsyncSync(fd); + const stats = fstatSync(fd); + assertRegularFileStats(stats, "artifact temp"); + if (stats.size !== bytes.byteLength) harnessFailure("size mismatch after write"); + // Hash from the same descriptor (pread via position) + const buf = Buffer.alloc(bytes.byteLength); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; + } + if (offset !== bytes.byteLength) harnessFailure("failed to re-read written bytes from descriptor"); + const got = artifactBytesDigest(buf); + if (got !== digest) harnessFailure("digest mismatch on same descriptor"); + closeSync(fd); + fd = null; + renameSync(tmpPath, target); + // Verify published object + return readArtifactBytes(dir, digest, bytes.byteLength); + } catch (err) { + if (fd !== null) { + try { closeSync(fd); } catch { /* ignore */ } + } + try { if (existsSync(tmpPath)) unlinkSync(tmpPath); } catch { /* ignore */ } + if (err instanceof ArtifactFsError) throw err; + harnessFailure(`artifact write failed: ${err instanceof Error ? err.message : String(err)}`); + } +} + +export interface ReadArtifactOptions { + expectedByteCount?: number; + /** + * Recompute content digest from descriptor bytes. + * Defaults to artifact-bytes domain. Contract artifacts pass their domain hash. + */ + contentDigest?: (bytes: Uint8Array) => string; +} + +/** Read and verify a content-addressed artifact from the trusted directory. */ +export function readArtifactBytes( + dir: TrustedArtifactDir, + digest: string, + expectedByteCountOrOpts?: number | ReadArtifactOptions, +): StoredArtifactBytes { + const opts: ReadArtifactOptions = + typeof expectedByteCountOrOpts === "number" + ? { expectedByteCount: expectedByteCountOrOpts } + : expectedByteCountOrOpts ?? {}; + const contentDigest = opts.contentDigest ?? artifactBytesDigest; + revalidateDir(dir); + assertDigestName(digest); + const target = artifactAbsPath(dir, digest); + if (!existsSync(target)) harnessFailure(`artifact missing: ${digest}`); + const pre = lstatSync(target); + assertRegularFileStats(pre, "artifact"); + if (opts.expectedByteCount !== undefined && pre.size !== opts.expectedByteCount) { + harnessFailure("artifact size mismatch before open"); + } + + let fd: number | null = null; + try { + fd = openSync(target, openFlags(O_RDONLY, true)); + const stats = fstatSync(fd); + assertRegularFileStats(stats, "artifact fd"); + if (stats.size !== pre.size || stats.ino !== pre.ino || stats.dev !== pre.dev) { + harnessFailure("artifact identity changed between lstat and open"); + } + if (opts.expectedByteCount !== undefined && stats.size !== opts.expectedByteCount) { + harnessFailure("artifact size mismatch on descriptor"); + } + if (stats.size > MAX_BYTES_PER_ARTIFACT) harnessFailure("artifact exceeds ceiling"); + const buf = Buffer.alloc(stats.size); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; + } + if (offset !== stats.size) harnessFailure("short read from artifact descriptor"); + const got = contentDigest(buf); + if (got !== digest) harnessFailure("artifact digest mismatch on descriptor"); + return { digest, bytes: new Uint8Array(buf), byteCount: stats.size }; + } finally { + if (fd !== null) closeSync(fd); + } +} + +/** Delete a content-addressed artifact after verifying it is a regular single-link file. */ +export function deleteArtifactBytes(dir: TrustedArtifactDir, digest: string): void { + revalidateDir(dir); + assertDigestName(digest); + const target = artifactAbsPath(dir, digest); + if (!existsSync(target)) return; + const stats = lstatSync(target); + assertRegularFileStats(stats, "artifact delete"); + unlinkSync(target); +} + +/** Contract fixture digests use the fixture domain; store under that digest name with raw bytes. */ +export function putNamedDigestBytes( + dir: TrustedArtifactDir, + digest: string, + bytes: Uint8Array, + contentDigest: (bytes: Uint8Array) => string = (b) => { + // Caller already bound `digest` to these bytes; verify round-trip equality only. + void b; + return digest; + }, +): StoredArtifactBytes { + revalidateDir(dir); + assertDigestName(digest); + if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) { + harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + if (contentDigest(bytes) !== digest) { + harnessFailure("named artifact content digest mismatch before write"); + } + const target = artifactAbsPath(dir, digest); + if (existsSync(target)) { + return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); + } + const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; + const tmpPath = join(dir.path, tmpName); + let fd: number | null = null; + try { + fd = openSync(tmpPath, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); + writeSync(fd, bytes); + fsyncSync(fd); + const stats = fstatSync(fd); + assertRegularFileStats(stats, "named artifact temp"); + if (stats.size !== bytes.byteLength) harnessFailure("size mismatch"); + const buf = Buffer.alloc(bytes.byteLength); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; + } + if (offset !== bytes.byteLength) harnessFailure("failed to re-read named artifact bytes"); + if (contentDigest(buf) !== digest) harnessFailure("named artifact digest mismatch on descriptor"); + closeSync(fd); + fd = null; + renameSync(tmpPath, target); + return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); + } catch (err) { + if (fd !== null) { + try { closeSync(fd); } catch { /* ignore */ } + } + try { if (existsSync(tmpPath)) unlinkSync(tmpPath); } catch { /* ignore */ } + if (err instanceof ArtifactFsError) throw err; + harnessFailure(`named artifact write failed: ${err instanceof Error ? err.message : String(err)}`); + } +} diff --git a/src/lab/artifacts/store.ts b/src/lab/artifacts/store.ts new file mode 100644 index 000000000..f5612b6c0 --- /dev/null +++ b/src/lab/artifacts/store.ts @@ -0,0 +1,240 @@ +import { MAX_BYTES_PER_ARTIFACT, MAX_AGGREGATE_ARTIFACT_BYTES, MAX_ARTIFACTS_PER_RUN, ARTIFACT_FILENAME_EXT, type ArtifactClass, type ContractArtifactClass } from "../constants"; +import { + artifactBytesDigest, + claimSourceManifestDigest, + fixtureDigest, + isSha256Hex, + jcsStringify, + scenarioManifestDigest, + suiteManifestDigest, +} from "../digest"; +import type { ArtifactRefV1, ClaimSourceManifestV1 } from "../events/types"; +import { artifactClassMediaType, validateClaimSourceManifest } from "../events/validate"; +import { + ArtifactFsError, + deleteArtifactBytes, + openTrustedArtifactDir, + putArtifactBytes, + putNamedDigestBytes, + readArtifactBytes, + type TrustedArtifactDir, +} from "./secure-fs"; +import { redactForArtifact } from "./sanitize"; + +export { ArtifactFsError, openTrustedArtifactDir }; +export type { TrustedArtifactDir }; + +export interface PutArtifactInput { + artifactClass: ArtifactClass; + /** Pre-redaction payload; sanitizer runs before hash/write. */ + payload: Uint8Array | string | unknown; + mediaType?: string; + redactionPolicy?: string; + /** For contract artifacts whose digest uses a domain other than artifact-bytes. */ + expectedDigest?: string; +} + +export interface ArtifactStore { + dir: TrustedArtifactDir; + put(input: PutArtifactInput): ArtifactRefV1; + get(digest: string, expectedByteCount?: number): Uint8Array; + getVerified(digest: string, expectedByteCount?: number): { bytes: Uint8Array; digest: string }; + remove(digest: string): void; +} + +function toBytes(payload: Uint8Array | string | unknown): Uint8Array { + if (payload instanceof Uint8Array) return payload; + if (typeof payload === "string") return new TextEncoder().encode(payload); + return new TextEncoder().encode(jcsStringify(payload)); +} + +export function createArtifactStore(artifactsDir: string): ArtifactStore { + const dir = openTrustedArtifactDir(artifactsDir); + let aggregateBytes = 0; + let putCount = 0; + + return { + dir, + put(input: PutArtifactInput): ArtifactRefV1 { + if (putCount >= MAX_ARTIFACTS_PER_RUN) { + throw new ArtifactFsError("budget_exhausted", "maximum artifacts per run exceeded"); + } + if (input.payload instanceof Uint8Array && input.payload.byteLength > MAX_BYTES_PER_ARTIFACT) { + throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + if (typeof input.payload === "string" && new TextEncoder().encode(input.payload).byteLength > MAX_BYTES_PER_ARTIFACT) { + throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + const redacted = redactForArtifact(input.artifactClass, input.payload); + const bytes = toBytes(redacted); + if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) { + throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + if (aggregateBytes + bytes.byteLength > MAX_AGGREGATE_ARTIFACT_BYTES) { + throw new ArtifactFsError("budget_exhausted", "aggregate artifact ceiling exceeded"); + } + + let stored; + if (isContractClass(input.artifactClass)) { + const contractClass = input.artifactClass; + const digest = input.expectedDigest ?? computeContractDigest(contractClass, bytes, redacted); + if (input.expectedDigest && digest !== input.expectedDigest) { + throw new ArtifactFsError("harness_failure", "contract artifact digest mismatch"); + } + const contentDigest = (b: Uint8Array) => + computeContractDigest(contractClass, b, JSON.parse(new TextDecoder().decode(b))); + // Fixtures hash raw bytes; JSON contract manifests hash parsed JCS object. + const hashFn = + contractClass === "fixture" + ? (b: Uint8Array) => fixtureDigest(b) + : contentDigest; + if (hashFn(bytes) !== digest) { + throw new ArtifactFsError("harness_failure", "contract artifact preimage digest mismatch"); + } + stored = putNamedDigestBytes(dir, digest, bytes, hashFn); + } else { + stored = putArtifactBytes(dir, bytes, input.expectedDigest); + } + + putCount += 1; + aggregateBytes += stored.byteCount; + return { + digest: stored.digest, + mediaType: input.mediaType ?? artifactClassMediaType(input.artifactClass), + byteCount: stored.byteCount, + redactionPolicy: input.redactionPolicy ?? defaultRedactionPolicy(input.artifactClass), + relativePath: `${stored.digest}${ARTIFACT_FILENAME_EXT}`, + artifactClass: input.artifactClass, + }; + }, + get(digest: string, expectedByteCount?: number): Uint8Array { + return this.getVerified(digest, expectedByteCount).bytes; + }, + getVerified(digest: string, expectedByteCount?: number) { + const candidates: Array<(b: Uint8Array) => string> = [ + artifactBytesDigest, + fixtureDigest, + (b) => { + try { + return scenarioManifestDigest(JSON.parse(new TextDecoder().decode(b))); + } catch { + return ""; + } + }, + (b) => { + try { + return suiteManifestDigest(JSON.parse(new TextDecoder().decode(b))); + } catch { + return ""; + } + }, + (b) => { + try { + return claimSourceManifestDigest(JSON.parse(new TextDecoder().decode(b))); + } catch { + return ""; + } + }, + ]; + let lastErr: unknown; + for (const contentDigest of candidates) { + try { + const got = readArtifactBytes(dir, digest, { expectedByteCount, contentDigest }); + if (got.digest === digest) return { bytes: got.bytes, digest: got.digest }; + } catch (err) { + lastErr = err; + } + } + throw lastErr instanceof Error + ? lastErr + : new ArtifactFsError("harness_failure", "artifact digest verification failed"); + }, + remove(digest: string): void { + deleteArtifactBytes(dir, digest); + }, + }; +} + +function isContractClass(c: ArtifactClass): c is ContractArtifactClass { + return ( + c === "scenario_manifest" || + c === "suite_manifest" || + c === "fixture" || + c === "claim_source_manifest" + ); +} + +function computeContractDigest( + artifactClass: ContractArtifactClass, + bytes: Uint8Array, + redacted: unknown, +): string { + switch (artifactClass) { + case "fixture": + return fixtureDigest(bytes); + case "scenario_manifest": + return scenarioManifestDigest( + typeof redacted === "object" && redacted ? (redacted as Record) : JSON.parse(new TextDecoder().decode(bytes)), + ); + case "suite_manifest": + return suiteManifestDigest( + typeof redacted === "object" && redacted ? (redacted as Record) : JSON.parse(new TextDecoder().decode(bytes)), + ); + case "claim_source_manifest": { + const parsed = typeof redacted === "object" && redacted + ? redacted + : JSON.parse(new TextDecoder().decode(bytes)); + return claimSourceManifestDigest(validateClaimSourceManifest(parsed).manifest); + } + default: { + const _never: never = artifactClass; + return _never; + } + } +} + +function defaultRedactionPolicy(artifactClass: ArtifactClass): string { + switch (artifactClass) { + case "scenario_manifest": + case "suite_manifest": + case "fixture": + case "claim_source_manifest": + return "contract_canonical_v1"; + default: + return "sanitized_evidence_v1"; + } +} + +export function putClaimSourceManifest( + store: ArtifactStore, + manifest: ClaimSourceManifestV1, +): ArtifactRefV1 { + const { manifest: validated, digest } = validateClaimSourceManifest(manifest); + return store.put({ + artifactClass: "claim_source_manifest", + payload: validated, + expectedDigest: digest, + }); +} + +export function loadClaimSourceManifest( + store: ArtifactStore, + digest: string, + expected: { subjectId: string; capability: string }, +): { manifest: ClaimSourceManifestV1; corruption?: string } { + if (!isSha256Hex(digest)) return { manifest: null as unknown as ClaimSourceManifestV1, corruption: "invalid digest" }; + try { + const bytes = store.get(digest); + const parsed = JSON.parse(new TextDecoder().decode(bytes)); + const { manifest, digest: recomputed } = validateClaimSourceManifest(parsed); + if (recomputed !== digest) return { manifest, corruption: "claim-source digest mismatch" }; + if (manifest.subjectId !== expected.subjectId) return { manifest, corruption: "claim-source subjectId mismatch" }; + if (manifest.capability !== expected.capability) return { manifest, corruption: "claim-source capability mismatch" }; + return { manifest }; + } catch (err) { + return { + manifest: null as unknown as ClaimSourceManifestV1, + corruption: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/lab/constants.ts b/src/lab/constants.ts new file mode 100644 index 000000000..ac7e851a6 --- /dev/null +++ b/src/lab/constants.ts @@ -0,0 +1,107 @@ +/** CL-02 projection / ledger constants frozen from CL-00 contracts. */ + +export const LAB_EVENT_SCHEMA_VERSION = 1; +export const LAB_PROJECTION_SPEC_VERSION = "cl-02.v1"; +export const LAB_PRODUCER = "opencodex-lab"; + +export const MAX_INVALIDATION_TARGETS = 1024; +export const MAX_BYTES_PER_ARTIFACT = 256 * 1024; +export const MAX_AGGREGATE_ARTIFACT_BYTES = 1024 * 1024; +export const MAX_ARTIFACTS_PER_RUN = 16; +export const MAX_SERIALIZED_EVENT_BYTES = 64 * 1024; +export const MAX_SANITIZED_STRING_FIELD = 4 * 1024; + +export const EVENT_KINDS = [ + "observation", + "claim_snapshot", + "invalidation", + "purge_tombstone", +] as const; + +export type LabEventKind = (typeof EVENT_KINDS)[number]; + +export const EVIDENCE_LAYERS = [ + "protocol_conformance", + "live_route_compatibility", + "task_effectiveness", +] as const; + +export type EvidenceLayer = (typeof EVIDENCE_LAYERS)[number]; + +export const VERDICTS = [ + "UNKNOWN", + "CLAIMED", + "PROBED", + "VERIFIED", + "DEGRADED", + "BLOCKED", + "UNSUPPORTED", +] as const; + +export type CompatibilityVerdict = (typeof VERDICTS)[number]; + +export const INVALIDATION_REASONS = [ + "harness_defect", + "fixture_defect", + "redaction_defect", + "integrity_defect", + "contract_artifact_missing", + "manual_correction", +] as const; + +export type InvalidationReason = (typeof INVALIDATION_REASONS)[number]; + +export const PURGE_ACTIONS = [ + "ledger", + "sqlite", + "artifact", + "scratch", + "export", +] as const; + +export type PurgeAction = (typeof PURGE_ACTIONS)[number]; + +export const CLAIM_SOURCE_KINDS = [ + "provider_config", + "provider_registry", + "cached_catalog", + "native_metadata", + "adapter_inference", +] as const; + +export type ClaimSourceKind = (typeof CLAIM_SOURCE_KINDS)[number]; + +export const ARTIFACT_CLASSES = [ + "scenario_manifest", + "suite_manifest", + "fixture", + "claim_source_manifest", + "assertion_report", + "request_shape", + "response_shape", + "event_trace", + "error_taxonomy", + "verifier_summary", +] as const; + +export type ArtifactClass = (typeof ARTIFACT_CLASSES)[number]; + +export const CONTRACT_ARTIFACT_CLASSES = [ + "scenario_manifest", + "suite_manifest", + "fixture", + "claim_source_manifest", +] as const; + +export type ContractArtifactClass = (typeof CONTRACT_ARTIFACT_CLASSES)[number]; + +export const EXECUTION_MODES = ["fixture", "live", "fabric"] as const; +export type ExecutionMode = (typeof EXECUTION_MODES)[number]; + +export const OUTCOMES = ["pass", "fail", "blocked", "inconclusive"] as const; +export type ObservationOutcome = (typeof OUTCOMES)[number]; + +export const CLAIM_POLARITIES = ["supported", "not_supported", "withdrawn"] as const; +export type ClaimPolarity = (typeof CLAIM_POLARITIES)[number]; + +export const ARTIFACT_FILENAME_EXT = ".bin"; diff --git a/src/lab/digest.ts b/src/lab/digest.ts new file mode 100644 index 000000000..c54b3b443 --- /dev/null +++ b/src/lab/digest.ts @@ -0,0 +1,64 @@ +import { createHash, createHmac } from "node:crypto"; +import { jcsStringify } from "./conformance/jcs"; +import { + fixtureDigest, + scenarioManifestDigest, + suiteManifestDigest, +} from "./conformance/digest"; + +export { fixtureDigest, scenarioManifestDigest, suiteManifestDigest, jcsStringify }; +export { jcsEqual } from "./conformance/jcs"; + +/** Domain-separated SHA-256: UTF-8(domain) || NUL || payload. */ +export function domainHash(domain: string, payload: Uint8Array | string): string { + const hash = createHash("sha256"); + hash.update(new TextEncoder().encode(`${domain}\0`)); + if (typeof payload === "string") hash.update(new TextEncoder().encode(payload)); + else hash.update(payload); + return hash.digest("hex"); +} + +export const LAB_DOMAIN = { + event: "ocx-lab:event:v1", + subject: "ocx-lab:subject:v1", + claimSourceManifest: "ocx-lab:claim-source-manifest:v1", + scenarioManifest: "ocx-lab:scenario-manifest:v1", + suiteManifest: "ocx-lab:suite-manifest:v1", + fixture: "ocx-lab:fixture:v1", + localFingerprint: "ocx-lab:local-fingerprint:v1", + artifactBytes: "ocx-lab:artifact-bytes:v1", +} as const; + +export function eventIdForPayload(eventWithoutId: unknown): string { + return domainHash(LAB_DOMAIN.event, jcsStringify(eventWithoutId)); +} + +export function subjectIdForSubject(subject: unknown): string { + return domainHash(LAB_DOMAIN.subject, jcsStringify(subject)); +} + +export function claimSourceManifestDigest(manifest: unknown): string { + return domainHash(LAB_DOMAIN.claimSourceManifest, jcsStringify(manifest)); +} + +/** Content digest for already-redacted artifact bytes (not contract fixture domain). */ +export function artifactBytesDigest(bytes: Uint8Array): string { + return domainHash(LAB_DOMAIN.artifactBytes, bytes); +} + +export function localFingerprint( + fieldName: string, + value: unknown, + salt: Uint8Array | string, +): string { + const key = typeof salt === "string" ? new TextEncoder().encode(salt) : salt; + const preimage = `${LAB_DOMAIN.localFingerprint}\0${fieldName}\0`; + return createHmac("sha256", key) + .update(new TextEncoder().encode(preimage)) + .update(new TextEncoder().encode(jcsStringify(value))) + .digest("hex"); +} + +export function isSha256Hex(value: string): boolean { + return /^[0-9a-f]{64}$/.test(value); +} diff --git a/src/lab/events/types.ts b/src/lab/events/types.ts new file mode 100644 index 000000000..69ed120db --- /dev/null +++ b/src/lab/events/types.ts @@ -0,0 +1,228 @@ +import type { + ArtifactClass, + ClaimPolarity, + ClaimSourceKind, + EvidenceLayer, + ExecutionMode, + InvalidationReason, + LabEventKind, + ObservationOutcome, + PurgeAction, +} from "../constants"; + +export interface LabProducer { + name: string; + version: string; +} + +export interface ProtocolSubjectV1 { + subjectSchemaVersion: 1; + subjectKind: "protocol"; + opencodexCompatibilityVersion: string; + effectiveAdapter: string; + inboundProtocol: string; + upstreamProtocol: string; + surface: string; + behaviorFingerprint: string; +} + +export interface RouteDependencyV1 { + role: string; + providerId: string; + providerInstanceFingerprint: string; + clientModelId: string; + upstreamModelId: string; + effectiveAdapter: string; + upstreamProtocol: string; + endpointFingerprint: string; + behaviorFingerprint: string; +} + +export interface RouteSubjectV1 { + subjectSchemaVersion: 1; + subjectKind: "route"; + providerId: string; + providerInstanceFingerprint: string; + clientModelId: string; + upstreamModelId: string; + effectiveAdapter: string; + inboundProtocol: string; + upstreamProtocol: string; + surface: string; + opencodexCompatibilityVersion: string; + behaviorFingerprint: string; + endpointFingerprint: string; + dependencies: RouteDependencyV1[]; +} + +export interface TaskSubjectV1 { + subjectSchemaVersion: 1; + subjectKind: "task"; + routeSubject: RouteSubjectV1; + taskClassId: string; + taskClassVersion: string; + taskFixtureDigest: string; + verifierManifestDigest: string; + fabricCompatibilityVersion: string; + sandboxProfileDigest: string; +} + +export type EvidenceSubjectV1 = ProtocolSubjectV1 | RouteSubjectV1 | TaskSubjectV1; + +export interface ArtifactRefV1 { + digest: string; + mediaType: string; + byteCount: number; + redactionPolicy: string; + /** Digest-derived relative name only (e.g. `.bin`). */ + relativePath: string; + artifactClass: ArtifactClass; +} + +export interface AssertionRecordV1 { + id: string; + operator: string; + required: boolean; + passed: boolean; + expectedSummary: string; + observedSummary: string; + reason?: string; +} + +export interface FailureRecordV1 { + class: string; + code: string; + retryable: boolean; + attribution: "opencodex" | "route" | "environment" | "harness"; +} + +export interface LabEventBase { + schemaVersion: number; + eventId: string; + eventKind: LabEventKind; + recordedAt: number; + producer: string; + producerVersion: string; +} + +export interface ObservationEvent extends LabEventBase { + eventKind: "observation"; + evidenceLayer: EvidenceLayer; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + fixtureDigests: string[]; + subject: EvidenceSubjectV1; + subjectId: string; + startedAt: number; + completedAt: number; + executionMode: ExecutionMode; + attempt: number; + limits: Record; + outcome: ObservationOutcome; + assertions: AssertionRecordV1[]; + failure?: FailureRecordV1; + expectedFailure?: Record; + environment: Record; + artifactRefs: ArtifactRefV1[]; + sourceRefs?: string[]; +} + +export interface ClaimSnapshotEvent extends LabEventBase { + eventKind: "claim_snapshot"; + evidenceLayer: "live_route_compatibility"; + subject: RouteSubjectV1; + subjectId: string; + capability: string; + polarity: ClaimPolarity; + sourceManifestDigest: string; + sourceEventIds: string[]; + supersedes: string[]; + effectiveAt: number; +} + +export interface InvalidationEvent extends LabEventBase { + eventKind: "invalidation"; + targetEventIds: string[]; + reason: InvalidationReason; +} + +export interface PurgeTombstoneEvent extends LabEventBase { + eventKind: "purge_tombstone"; + targetEventIds: string[]; + targetArtifactDigests: string[]; + reason: "sensitive_evidence"; + purgeActions: PurgeAction[]; +} + +export type LabEvent = + | ObservationEvent + | ClaimSnapshotEvent + | InvalidationEvent + | PurgeTombstoneEvent; + +export type ClaimCapabilityFactsV1 = { + contextWindow?: number; + inputModalities?: string[]; + reasoningEfforts?: string[]; + catalogCapabilityNames?: string[]; + serviceTier?: string[]; + toolCapable?: boolean; + parallelToolCalls?: boolean; + endpointLocality?: "local" | "private" | "unknown"; + canonicalOpenAiForward?: boolean; +}; + +export interface ClaimSourceV1 { + kind: ClaimSourceKind; + revision: string | null; + facts: ClaimCapabilityFactsV1; +} + +export interface RouteCapabilityEvidenceV1 { + contextWindow?: number; + tools?: boolean | "unknown"; + image?: boolean | "unknown"; + structuredOutput?: boolean | "unknown"; + reasoningEfforts?: string[]; + serviceTier?: string | "unknown"; + localOnly?: boolean | "unknown"; + remoteAllowed?: boolean | "unknown"; + encryptedCodexTasks?: boolean | "unknown"; +} + +export interface ClaimSourceManifestV1 { + schemaVersion: 1; + subjectId: string; + providerId: string; + clientModelId: string; + capability: string; + sources: ClaimSourceV1[]; + resolvedEvidence: RouteCapabilityEvidenceV1; +} + +export interface LedgerCorruption { + kind: + | "malformed_line" + | "partial_line" + | "duplicate_event" + | "invalid_event" + | "invalid_reference" + | "missing_artifact" + | "artifact_mismatch" + | "claim_corruption" + | "empty_ledger"; + lineNumber?: number; + eventId?: string; + detail: string; +} + +export interface ReplayResult { + events: LabEvent[]; + corruptions: LedgerCorruption[]; + validLineCount: number; + totalLineCount: number; +} diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts new file mode 100644 index 000000000..bcccc7d54 --- /dev/null +++ b/src/lab/events/validate.ts @@ -0,0 +1,676 @@ +import { + ARTIFACT_CLASSES, + ARTIFACT_FILENAME_EXT, + CLAIM_POLARITIES, + CLAIM_SOURCE_KINDS, + EVIDENCE_LAYERS, + EVENT_KINDS, + EXECUTION_MODES, + INVALIDATION_REASONS, + LAB_EVENT_SCHEMA_VERSION, + MAX_INVALIDATION_TARGETS, + MAX_SANITIZED_STRING_FIELD, + MAX_SERIALIZED_EVENT_BYTES, + OUTCOMES, + PURGE_ACTIONS, + type ArtifactClass, + type ClaimSourceKind, + type EvidenceLayer, + type LabEventKind, +} from "../constants"; +import { + claimSourceManifestDigest, + eventIdForPayload, + isSha256Hex, + jcsStringify, + subjectIdForSubject, +} from "../digest"; +import type { + ArtifactRefV1, + ClaimCapabilityFactsV1, + ClaimSnapshotEvent, + ClaimSourceManifestV1, + ClaimSourceV1, + EvidenceSubjectV1, + InvalidationEvent, + LabEvent, + ObservationEvent, + ProtocolSubjectV1, + PurgeTombstoneEvent, + RouteCapabilityEvidenceV1, + RouteSubjectV1, + TaskSubjectV1, +} from "./types"; + +export class LabValidationError extends Error { + readonly code: string; + constructor(code: string, message: string) { + super(message); + this.name = "LabValidationError"; + this.code = code; + } +} + +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function assertString(value: unknown, field: string, max = MAX_SANITIZED_STRING_FIELD): string { + if (typeof value !== "string") throw new LabValidationError("invalid_type", `${field} must be string`); + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes > max) throw new LabValidationError("field_too_large", `${field} exceeds ${max} bytes`); + if (value.includes("\0")) throw new LabValidationError("nul_forbidden", `${field} contains NUL`); + return value; +} + +function assertIntMs(value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + throw new LabValidationError("invalid_timestamp", `${field} must be UTC epoch milliseconds`); + } + return value; +} + +function assertClosed(value: unknown, field: string, allowed: readonly T[]): T { + if (typeof value !== "string" || !(allowed as readonly string[]).includes(value)) { + throw new LabValidationError("closed_set", `${field} must be one of ${allowed.join("|")}`); + } + return value as T; +} + +function utf8LexLess(a: string, b: string): boolean { + const ae = new TextEncoder().encode(a); + const be = new TextEncoder().encode(b); + const n = Math.min(ae.length, be.length); + for (let i = 0; i < n; i++) { + if (ae[i]! < be[i]!) return true; + if (ae[i]! > be[i]!) return false; + } + return ae.length < be.length; +} + +/** Sorted unique lowercase SHA-256 hex IDs; rejects duplicates / unsorted / uppercase. */ +export function validateSortedUniqueHexIds( + ids: unknown, + field: string, + opts: { nonEmpty?: boolean; max?: number } = {}, +): string[] { + if (!Array.isArray(ids)) throw new LabValidationError("invalid_type", `${field} must be array`); + if (opts.nonEmpty && ids.length === 0) { + throw new LabValidationError("empty_targets", `${field} must be non-empty`); + } + if (opts.max !== undefined && ids.length > opts.max) { + throw new LabValidationError("too_many_targets", `${field} exceeds ${opts.max}`); + } + const out: string[] = []; + for (let i = 0; i < ids.length; i++) { + const id = ids[i]; + if (typeof id !== "string" || !isSha256Hex(id)) { + throw new LabValidationError("invalid_id", `${field}[${i}] must be lowercase sha256 hex`); + } + if (i > 0) { + const prev = out[i - 1]!; + if (id === prev) throw new LabValidationError("duplicate_id", `${field} contains duplicates`); + if (!utf8LexLess(prev, id)) { + throw new LabValidationError("unsorted_ids", `${field} must be UTF-8 lexicographically sorted`); + } + } + out.push(id); + } + return out; +} + +function validateArtifactRef(raw: unknown, index: number): ArtifactRefV1 { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_artifact_ref", `artifactRefs[${index}]`); + const digest = assertString(raw.digest, `artifactRefs[${index}].digest`); + if (!isSha256Hex(digest)) throw new LabValidationError("invalid_digest", `artifactRefs[${index}].digest`); + const mediaType = assertString(raw.mediaType, `artifactRefs[${index}].mediaType`, 256); + if (typeof raw.byteCount !== "number" || !Number.isInteger(raw.byteCount) || raw.byteCount < 0) { + throw new LabValidationError("invalid_byte_count", `artifactRefs[${index}].byteCount`); + } + const redactionPolicy = assertString(raw.redactionPolicy, `artifactRefs[${index}].redactionPolicy`, 256); + const relativePath = assertString(raw.relativePath, `artifactRefs[${index}].relativePath`, 128); + if (relativePath !== `${digest}${ARTIFACT_FILENAME_EXT}`) { + throw new LabValidationError("invalid_relative_path", `artifactRefs[${index}].relativePath must be digest-derived`); + } + if (relativePath.includes("/") || relativePath.includes("\\") || relativePath.includes("..")) { + throw new LabValidationError("path_traversal", `artifactRefs[${index}].relativePath`); + } + const artifactClass = assertClosed(raw.artifactClass, `artifactRefs[${index}].artifactClass`, ARTIFACT_CLASSES); + return { digest, mediaType, byteCount: raw.byteCount, redactionPolicy, relativePath, artifactClass }; +} + +function validateProtocolSubject(raw: Record): ProtocolSubjectV1 { + if (raw.subjectSchemaVersion !== 1 || raw.subjectKind !== "protocol") { + throw new LabValidationError("invalid_subject", "protocol subject schema/kind mismatch"); + } + return { + subjectSchemaVersion: 1, + subjectKind: "protocol", + opencodexCompatibilityVersion: assertString(raw.opencodexCompatibilityVersion, "opencodexCompatibilityVersion"), + effectiveAdapter: assertString(raw.effectiveAdapter, "effectiveAdapter"), + inboundProtocol: assertString(raw.inboundProtocol, "inboundProtocol"), + upstreamProtocol: assertString(raw.upstreamProtocol, "upstreamProtocol"), + surface: assertString(raw.surface, "surface"), + behaviorFingerprint: assertString(raw.behaviorFingerprint, "behaviorFingerprint"), + }; +} + +function validateRouteSubject(raw: Record): RouteSubjectV1 { + if (raw.subjectSchemaVersion !== 1 || raw.subjectKind !== "route") { + throw new LabValidationError("invalid_subject", "route subject schema/kind mismatch"); + } + if (!Array.isArray(raw.dependencies)) { + throw new LabValidationError("invalid_subject", "dependencies must be array"); + } + const dependencies = raw.dependencies.map((dep, i) => { + if (!isPlainObject(dep)) throw new LabValidationError("invalid_dependency", `dependencies[${i}]`); + return { + role: assertString(dep.role, `dependencies[${i}].role`), + providerId: assertString(dep.providerId, `dependencies[${i}].providerId`), + providerInstanceFingerprint: assertString(dep.providerInstanceFingerprint, `dependencies[${i}].providerInstanceFingerprint`), + clientModelId: assertString(dep.clientModelId, `dependencies[${i}].clientModelId`), + upstreamModelId: assertString(dep.upstreamModelId, `dependencies[${i}].upstreamModelId`), + effectiveAdapter: assertString(dep.effectiveAdapter, `dependencies[${i}].effectiveAdapter`), + upstreamProtocol: assertString(dep.upstreamProtocol, `dependencies[${i}].upstreamProtocol`), + endpointFingerprint: assertString(dep.endpointFingerprint, `dependencies[${i}].endpointFingerprint`), + behaviorFingerprint: assertString(dep.behaviorFingerprint, `dependencies[${i}].behaviorFingerprint`), + }; + }); + return { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: assertString(raw.providerId, "providerId"), + providerInstanceFingerprint: assertString(raw.providerInstanceFingerprint, "providerInstanceFingerprint"), + clientModelId: assertString(raw.clientModelId, "clientModelId"), + upstreamModelId: assertString(raw.upstreamModelId, "upstreamModelId"), + effectiveAdapter: assertString(raw.effectiveAdapter, "effectiveAdapter"), + inboundProtocol: assertString(raw.inboundProtocol, "inboundProtocol"), + upstreamProtocol: assertString(raw.upstreamProtocol, "upstreamProtocol"), + surface: assertString(raw.surface, "surface"), + opencodexCompatibilityVersion: assertString(raw.opencodexCompatibilityVersion, "opencodexCompatibilityVersion"), + behaviorFingerprint: assertString(raw.behaviorFingerprint, "behaviorFingerprint"), + endpointFingerprint: assertString(raw.endpointFingerprint, "endpointFingerprint"), + dependencies, + }; +} + +function validateTaskSubject(raw: Record): TaskSubjectV1 { + if (raw.subjectSchemaVersion !== 1 || raw.subjectKind !== "task") { + throw new LabValidationError("invalid_subject", "task subject schema/kind mismatch"); + } + if (!isPlainObject(raw.routeSubject)) { + throw new LabValidationError("invalid_subject", "task.routeSubject required"); + } + return { + subjectSchemaVersion: 1, + subjectKind: "task", + routeSubject: validateRouteSubject(raw.routeSubject), + taskClassId: assertString(raw.taskClassId, "taskClassId"), + taskClassVersion: assertString(raw.taskClassVersion, "taskClassVersion"), + taskFixtureDigest: assertString(raw.taskFixtureDigest, "taskFixtureDigest"), + verifierManifestDigest: assertString(raw.verifierManifestDigest, "verifierManifestDigest"), + fabricCompatibilityVersion: assertString(raw.fabricCompatibilityVersion, "fabricCompatibilityVersion"), + sandboxProfileDigest: assertString(raw.sandboxProfileDigest, "sandboxProfileDigest"), + }; +} + +export function validateSubject(raw: unknown, layer: EvidenceLayer): EvidenceSubjectV1 { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_subject", "subject must be object"); + const kind = raw.subjectKind; + if (layer === "protocol_conformance") { + if (kind !== "protocol") throw new LabValidationError("layer_subject_mismatch", "protocol layer requires protocol subject"); + return validateProtocolSubject(raw); + } + if (layer === "live_route_compatibility") { + if (kind !== "route") throw new LabValidationError("layer_subject_mismatch", "live layer requires route subject"); + return validateRouteSubject(raw); + } + if (layer === "task_effectiveness") { + if (kind !== "task") throw new LabValidationError("layer_subject_mismatch", "task layer requires task subject"); + return validateTaskSubject(raw); + } + const _exhaustive: never = layer; + throw new LabValidationError("unknown_layer", String(_exhaustive)); +} + +function stripEventId(event: Record): Record { + const { eventId: _omit, ...rest } = event; + return rest; +} + +function enforceEventId(event: LabEvent): void { + const recomputed = eventIdForPayload(stripEventId(event as unknown as Record)); + if (event.eventId !== recomputed) { + throw new LabValidationError("event_id_mismatch", `eventId mismatch: got ${event.eventId}, expected ${recomputed}`); + } +} + +function enforceSerializedSize(event: LabEvent): void { + const bytes = new TextEncoder().encode(jcsStringify(event)).byteLength; + if (bytes > MAX_SERIALIZED_EVENT_BYTES) { + throw new LabValidationError("event_too_large", `serialized event exceeds ${MAX_SERIALIZED_EVENT_BYTES} bytes`); + } +} + +function validateObservation(raw: Record): ObservationEvent { + const evidenceLayer = assertClosed(raw.evidenceLayer, "evidenceLayer", EVIDENCE_LAYERS); + const subject = validateSubject(raw.subject, evidenceLayer); + const subjectId = assertString(raw.subjectId, "subjectId"); + if (subjectId !== subjectIdForSubject(subject)) { + throw new LabValidationError("subject_id_mismatch", "subjectId does not match subject"); + } + if (!Array.isArray(raw.fixtureDigests) || raw.fixtureDigests.length === 0) { + throw new LabValidationError("missing_fixture_digests", "fixtureDigests required"); + } + const fixtureDigests = raw.fixtureDigests.map((d, i) => { + const digest = assertString(d, `fixtureDigests[${i}]`); + if (!isSha256Hex(digest)) throw new LabValidationError("invalid_digest", `fixtureDigests[${i}]`); + return digest; + }); + const scenarioManifestDigest = assertString(raw.scenarioManifestDigest, "scenarioManifestDigest"); + const suiteManifestDigest = assertString(raw.suiteManifestDigest, "suiteManifestDigest"); + if (!isSha256Hex(scenarioManifestDigest) || !isSha256Hex(suiteManifestDigest)) { + throw new LabValidationError("invalid_digest", "scenario/suite manifest digest"); + } + if (!Array.isArray(raw.assertions)) throw new LabValidationError("invalid_assertions", "assertions"); + if (!Array.isArray(raw.artifactRefs)) throw new LabValidationError("invalid_artifact_refs", "artifactRefs"); + const event: ObservationEvent = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventId: assertString(raw.eventId, "eventId"), + eventKind: "observation", + recordedAt: assertIntMs(raw.recordedAt, "recordedAt"), + producer: assertString(raw.producer, "producer"), + producerVersion: assertString(raw.producerVersion, "producerVersion"), + evidenceLayer, + scenarioId: assertString(raw.scenarioId, "scenarioId"), + scenarioVersion: assertString(raw.scenarioVersion, "scenarioVersion"), + scenarioManifestDigest, + suiteId: assertString(raw.suiteId, "suiteId"), + suiteVersion: assertString(raw.suiteVersion, "suiteVersion"), + suiteManifestDigest, + fixtureDigests, + subject, + subjectId, + startedAt: assertIntMs(raw.startedAt, "startedAt"), + completedAt: assertIntMs(raw.completedAt, "completedAt"), + executionMode: assertClosed(raw.executionMode, "executionMode", EXECUTION_MODES), + attempt: typeof raw.attempt === "number" && Number.isInteger(raw.attempt) && raw.attempt >= 1 + ? raw.attempt + : (() => { throw new LabValidationError("invalid_attempt", "attempt"); })(), + limits: isPlainObject(raw.limits) ? raw.limits as Record : (() => { + throw new LabValidationError("invalid_limits", "limits"); + })(), + outcome: assertClosed(raw.outcome, "outcome", OUTCOMES), + assertions: raw.assertions as ObservationEvent["assertions"], + environment: isPlainObject(raw.environment) ? raw.environment : (() => { + throw new LabValidationError("invalid_environment", "environment"); + })(), + artifactRefs: raw.artifactRefs.map(validateArtifactRef), + }; + if (raw.failure !== undefined) { + if (!isPlainObject(raw.failure)) throw new LabValidationError("invalid_failure", "failure"); + event.failure = { + class: assertString(raw.failure.class, "failure.class"), + code: assertString(raw.failure.code, "failure.code"), + retryable: raw.failure.retryable === true, + attribution: assertClosed(raw.failure.attribution, "failure.attribution", [ + "opencodex", + "route", + "environment", + "harness", + ] as const), + }; + } + if (raw.expectedFailure !== undefined) { + if (!isPlainObject(raw.expectedFailure)) throw new LabValidationError("invalid_expected_failure", "expectedFailure"); + event.expectedFailure = raw.expectedFailure; + } + if (raw.sourceRefs !== undefined) { + if (!Array.isArray(raw.sourceRefs)) throw new LabValidationError("invalid_source_refs", "sourceRefs"); + event.sourceRefs = raw.sourceRefs.map((r, i) => assertString(r, `sourceRefs[${i}]`)); + } + if (event.completedAt < event.startedAt) { + throw new LabValidationError("invalid_time_range", "completedAt < startedAt"); + } + return event; +} + +function validateClaimSnapshot(raw: Record): ClaimSnapshotEvent { + if (raw.evidenceLayer !== "live_route_compatibility") { + throw new LabValidationError("invalid_claim_layer", "claims exist only for live_route_compatibility"); + } + const subject = validateSubject(raw.subject, "live_route_compatibility") as RouteSubjectV1; + const subjectId = assertString(raw.subjectId, "subjectId"); + if (subjectId !== subjectIdForSubject(subject)) { + throw new LabValidationError("subject_id_mismatch", "subjectId does not match subject"); + } + const sourceManifestDigest = assertString(raw.sourceManifestDigest, "sourceManifestDigest"); + if (!isSha256Hex(sourceManifestDigest)) { + throw new LabValidationError("invalid_digest", "sourceManifestDigest"); + } + if (!Array.isArray(raw.sourceEventIds) || !Array.isArray(raw.supersedes)) { + throw new LabValidationError("invalid_claim_lists", "sourceEventIds/supersedes"); + } + return { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventId: assertString(raw.eventId, "eventId"), + eventKind: "claim_snapshot", + recordedAt: assertIntMs(raw.recordedAt, "recordedAt"), + producer: assertString(raw.producer, "producer"), + producerVersion: assertString(raw.producerVersion, "producerVersion"), + evidenceLayer: "live_route_compatibility", + subject, + subjectId, + capability: assertString(raw.capability, "capability"), + polarity: assertClosed(raw.polarity, "polarity", CLAIM_POLARITIES), + sourceManifestDigest, + sourceEventIds: raw.sourceEventIds.map((id, i) => { + const s = assertString(id, `sourceEventIds[${i}]`); + if (s.length > 0 && !isSha256Hex(s)) throw new LabValidationError("invalid_id", `sourceEventIds[${i}]`); + return s; + }), + supersedes: validateSortedUniqueHexIds(raw.supersedes, "supersedes", { nonEmpty: false }), + effectiveAt: assertIntMs(raw.effectiveAt, "effectiveAt"), + }; +} + +function validateInvalidation(raw: Record): InvalidationEvent { + return { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventId: assertString(raw.eventId, "eventId"), + eventKind: "invalidation", + recordedAt: assertIntMs(raw.recordedAt, "recordedAt"), + producer: assertString(raw.producer, "producer"), + producerVersion: assertString(raw.producerVersion, "producerVersion"), + targetEventIds: validateSortedUniqueHexIds(raw.targetEventIds, "targetEventIds", { + nonEmpty: true, + max: MAX_INVALIDATION_TARGETS, + }), + reason: assertClosed(raw.reason, "reason", INVALIDATION_REASONS), + }; +} + +function validatePurge(raw: Record): PurgeTombstoneEvent { + const targetEventIds = validateSortedUniqueHexIds(raw.targetEventIds ?? [], "targetEventIds", { + nonEmpty: false, + max: MAX_INVALIDATION_TARGETS, + }); + const targetArtifactDigests = validateSortedUniqueHexIds( + raw.targetArtifactDigests ?? [], + "targetArtifactDigests", + { nonEmpty: false, max: MAX_INVALIDATION_TARGETS }, + ); + if (targetEventIds.length === 0 && targetArtifactDigests.length === 0) { + throw new LabValidationError("empty_purge_targets", "at least one purge target required"); + } + if (raw.reason !== "sensitive_evidence") { + throw new LabValidationError("invalid_purge_reason", "reason must be sensitive_evidence"); + } + if (!Array.isArray(raw.purgeActions) || raw.purgeActions.length === 0) { + throw new LabValidationError("empty_purge_actions", "purgeActions required"); + } + const purgeActions = raw.purgeActions.map((a, i) => + assertClosed(a, `purgeActions[${i}]`, PURGE_ACTIONS), + ); + for (let i = 0; i < purgeActions.length; i++) { + for (let j = i + 1; j < purgeActions.length; j++) { + if (purgeActions[i] === purgeActions[j]) { + throw new LabValidationError("duplicate_purge_action", "purgeActions must be unique"); + } + } + } + const sortedActions = [...purgeActions].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + for (let i = 0; i < purgeActions.length; i++) { + if (purgeActions[i] !== sortedActions[i]) { + throw new LabValidationError("unsorted_purge_actions", "purgeActions must be sorted"); + } + } + return { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventId: assertString(raw.eventId, "eventId"), + eventKind: "purge_tombstone", + recordedAt: assertIntMs(raw.recordedAt, "recordedAt"), + producer: assertString(raw.producer, "producer"), + producerVersion: assertString(raw.producerVersion, "producerVersion"), + targetEventIds, + targetArtifactDigests, + reason: "sensitive_evidence", + purgeActions, + }; +} + +/** Structural validation of a parsed JSON object into a LabEvent (fail-closed). */ +export function validateLabEvent(raw: unknown): LabEvent { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_event", "event must be object"); + if (raw.schemaVersion !== LAB_EVENT_SCHEMA_VERSION) { + throw new LabValidationError("schema_version", "unsupported schemaVersion"); + } + const kind = assertClosed(raw.eventKind, "eventKind", EVENT_KINDS) as LabEventKind; + let event: LabEvent; + switch (kind) { + case "observation": + event = validateObservation(raw); + break; + case "claim_snapshot": + event = validateClaimSnapshot(raw); + break; + case "invalidation": + event = validateInvalidation(raw); + break; + case "purge_tombstone": + event = validatePurge(raw); + break; + default: { + const _never: never = kind; + throw new LabValidationError("unknown_kind", String(_never)); + } + } + enforceEventId(event); + enforceSerializedSize(event); + return event; +} + +const FORBIDDEN_FACT_KEYS = new Set([ + "baseUrl", + "hostname", + "url", + "headers", + "authorization", + "apiKey", + "credential", + "account", + "email", + "path", + "prompt", + "repository", +]); + +const ALLOWED_FACT_KEYS = new Set([ + "contextWindow", + "inputModalities", + "reasoningEfforts", + "catalogCapabilityNames", + "serviceTier", + "toolCapable", + "parallelToolCalls", + "endpointLocality", + "canonicalOpenAiForward", +]); + +function validateFacts(raw: unknown, label: string): ClaimCapabilityFactsV1 { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_facts", label); + for (const key of Object.keys(raw)) { + if (FORBIDDEN_FACT_KEYS.has(key) || /url|host|header|secret|token|cookie|email|path/i.test(key)) { + throw new LabValidationError("forbidden_fact", `${label}.${key} forbidden`); + } + if (!ALLOWED_FACT_KEYS.has(key)) { + throw new LabValidationError("unknown_fact", `${label}.${key} unknown`); + } + } + const facts: ClaimCapabilityFactsV1 = {}; + if (raw.contextWindow !== undefined) { + if (typeof raw.contextWindow !== "number" || !Number.isInteger(raw.contextWindow) || raw.contextWindow < 0) { + throw new LabValidationError("invalid_facts", "contextWindow"); + } + facts.contextWindow = raw.contextWindow; + } + if (raw.inputModalities !== undefined) { + if (!Array.isArray(raw.inputModalities) || !raw.inputModalities.every((x) => typeof x === "string")) { + throw new LabValidationError("invalid_facts", "inputModalities"); + } + facts.inputModalities = raw.inputModalities as string[]; + } + if (raw.reasoningEfforts !== undefined) { + if (!Array.isArray(raw.reasoningEfforts) || !raw.reasoningEfforts.every((x) => typeof x === "string")) { + throw new LabValidationError("invalid_facts", "reasoningEfforts"); + } + facts.reasoningEfforts = raw.reasoningEfforts as string[]; + } + if (raw.catalogCapabilityNames !== undefined) { + if (!Array.isArray(raw.catalogCapabilityNames) || !raw.catalogCapabilityNames.every((x) => typeof x === "string")) { + throw new LabValidationError("invalid_facts", "catalogCapabilityNames"); + } + facts.catalogCapabilityNames = raw.catalogCapabilityNames as string[]; + } + if (raw.serviceTier !== undefined) { + if (!Array.isArray(raw.serviceTier) || !raw.serviceTier.every((x) => typeof x === "string")) { + throw new LabValidationError("invalid_facts", "serviceTier"); + } + facts.serviceTier = raw.serviceTier as string[]; + } + if (raw.toolCapable !== undefined) { + if (typeof raw.toolCapable !== "boolean") throw new LabValidationError("invalid_facts", "toolCapable"); + facts.toolCapable = raw.toolCapable; + } + if (raw.parallelToolCalls !== undefined) { + if (typeof raw.parallelToolCalls !== "boolean") throw new LabValidationError("invalid_facts", "parallelToolCalls"); + facts.parallelToolCalls = raw.parallelToolCalls; + } + if (raw.endpointLocality !== undefined) { + facts.endpointLocality = assertClosed(raw.endpointLocality, "endpointLocality", [ + "local", + "private", + "unknown", + ] as const); + } + if (raw.canonicalOpenAiForward !== undefined) { + if (typeof raw.canonicalOpenAiForward !== "boolean") { + throw new LabValidationError("invalid_facts", "canonicalOpenAiForward"); + } + facts.canonicalOpenAiForward = raw.canonicalOpenAiForward; + } + return facts; +} + +function validateResolvedEvidence(raw: unknown): RouteCapabilityEvidenceV1 { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_resolved", "resolvedEvidence"); + const out: RouteCapabilityEvidenceV1 = {}; + const copyBoolOrUnknown = (key: keyof RouteCapabilityEvidenceV1) => { + const v = raw[key as string]; + if (v === undefined) return; + if (v !== true && v !== false && v !== "unknown") { + throw new LabValidationError("invalid_resolved", String(key)); + } + (out as Record)[key] = v; + }; + if (raw.contextWindow !== undefined) { + if (typeof raw.contextWindow !== "number" || !Number.isInteger(raw.contextWindow)) { + throw new LabValidationError("invalid_resolved", "contextWindow"); + } + out.contextWindow = raw.contextWindow; + } + copyBoolOrUnknown("tools"); + copyBoolOrUnknown("image"); + copyBoolOrUnknown("structuredOutput"); + copyBoolOrUnknown("localOnly"); + copyBoolOrUnknown("remoteAllowed"); + copyBoolOrUnknown("encryptedCodexTasks"); + if (raw.reasoningEfforts !== undefined) { + if (!Array.isArray(raw.reasoningEfforts) || !raw.reasoningEfforts.every((x) => typeof x === "string")) { + throw new LabValidationError("invalid_resolved", "reasoningEfforts"); + } + out.reasoningEfforts = raw.reasoningEfforts as string[]; + } + if (raw.serviceTier !== undefined) { + if (typeof raw.serviceTier !== "string") throw new LabValidationError("invalid_resolved", "serviceTier"); + out.serviceTier = raw.serviceTier; + } + return out; +} + +/** Validate ClaimSourceManifestV1 and return canonical object + digest. */ +export function validateClaimSourceManifest(raw: unknown): { + manifest: ClaimSourceManifestV1; + digest: string; +} { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_claim_source", "manifest must be object"); + if (raw.schemaVersion !== 1) throw new LabValidationError("schema_version", "ClaimSourceManifest schemaVersion"); + if (!Array.isArray(raw.sources)) throw new LabValidationError("invalid_sources", "sources"); + const seen = new Set(); + const sources: ClaimSourceV1[] = []; + for (let i = 0; i < raw.sources.length; i++) { + const src = raw.sources[i]; + if (!isPlainObject(src)) throw new LabValidationError("invalid_source", `sources[${i}]`); + const kind = assertClosed(src.kind, `sources[${i}].kind`, CLAIM_SOURCE_KINDS); + if (seen.has(kind)) throw new LabValidationError("duplicate_source_kind", kind); + seen.add(kind); + if (src.revision !== null && typeof src.revision !== "string") { + throw new LabValidationError("invalid_revision", `sources[${i}].revision`); + } + sources.push({ + kind, + revision: src.revision as string | null, + facts: validateFacts(src.facts, `sources[${i}].facts`), + }); + } + // Closed kind order + for (let i = 1; i < sources.length; i++) { + const prevIdx = CLAIM_SOURCE_KINDS.indexOf(sources[i - 1]!.kind); + const curIdx = CLAIM_SOURCE_KINDS.indexOf(sources[i]!.kind); + if (curIdx <= prevIdx) { + throw new LabValidationError("source_order", "sources must follow closed kind order"); + } + } + const manifest: ClaimSourceManifestV1 = { + schemaVersion: 1, + subjectId: assertString(raw.subjectId, "subjectId"), + providerId: assertString(raw.providerId, "providerId"), + clientModelId: assertString(raw.clientModelId, "clientModelId"), + capability: assertString(raw.capability, "capability"), + sources, + resolvedEvidence: validateResolvedEvidence(raw.resolvedEvidence), + }; + if (!isSha256Hex(manifest.subjectId)) { + throw new LabValidationError("invalid_subject_id", "subjectId must be sha256 hex"); + } + return { manifest, digest: claimSourceManifestDigest(manifest) }; +} + +export function assignEventId & { eventId?: string }>( + eventWithoutId: T, +): T & { eventId: string } { + const { eventId: _ignored, ...rest } = eventWithoutId as T & { eventId?: string }; + const eventId = eventIdForPayload(rest); + return { ...rest, eventId } as T & { eventId: string }; +} + +export function artifactClassMediaType(artifactClass: ArtifactClass): string { + switch (artifactClass) { + case "scenario_manifest": + case "suite_manifest": + case "claim_source_manifest": + case "assertion_report": + case "request_shape": + case "response_shape": + case "event_trace": + case "error_taxonomy": + case "verifier_summary": + return "application/json"; + case "fixture": + return "application/octet-stream"; + default: { + const _never: never = artifactClass; + return _never; + } + } +} diff --git a/src/lab/index.ts b/src/lab/index.ts new file mode 100644 index 000000000..0415b840f --- /dev/null +++ b/src/lab/index.ts @@ -0,0 +1,15 @@ +export * from "./constants"; +export * from "./digest"; +export * from "./paths"; +export * from "./events/types"; +export * from "./events/validate"; +export * from "./artifacts/secure-fs"; +export * from "./artifacts/store"; +export * from "./artifacts/sanitize"; +export * from "./ledger/store"; +export * from "./ledger/invalidation"; +export * from "./ledger/purge"; +export * from "./projection/schema"; +export * from "./projection/verdicts"; +export * from "./projection/rebuild"; +export * from "./observe/from-conformance"; diff --git a/src/lab/ledger/invalidation.ts b/src/lab/ledger/invalidation.ts new file mode 100644 index 000000000..c8b2d317e --- /dev/null +++ b/src/lab/ledger/invalidation.ts @@ -0,0 +1,136 @@ +import type { + ClaimSnapshotEvent, + InvalidationEvent, + LabEvent, + LedgerCorruption, + ObservationEvent, + PurgeTombstoneEvent, +} from "../events/types"; +import { LabValidationError } from "../events/validate"; + +export interface InvalidationIndex { + /** eventId -> invalidation eventIds that targeted it */ + invalidatedBy: Map; + purgedEventIds: Set; + purgedArtifactDigests: Set; + corruptions: LedgerCorruption[]; +} + +/** + * Apply purge tombstones before ordinary invalidations. + * Invalidation target lists are atomic: any bad target rejects the whole event. + */ +export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex { + const purgedEventIds = new Set(); + const purgedArtifactDigests = new Set(); + const invalidatedBy = new Map(); + const corruptions: LedgerCorruption[] = []; + + const validEvidenceIds = new Map(); + + // First pass: record evidence positions; apply purges as encountered. + for (let i = 0; i < events.length; i++) { + const event = events[i]!; + if (event.eventKind === "observation" || event.eventKind === "claim_snapshot") { + validEvidenceIds.set(event.eventId, { kind: event.eventKind, index: i }); + continue; + } + if (event.eventKind === "purge_tombstone") { + applyPurge(event, purgedEventIds, purgedArtifactDigests); + } + } + + // Second pass: validate and apply invalidations against earlier evidence. + for (let i = 0; i < events.length; i++) { + const event = events[i]!; + if (event.eventKind !== "invalidation") continue; + try { + validateInvalidationTargets(event, events, i, validEvidenceIds, purgedEventIds); + for (const target of event.targetEventIds) { + const list = invalidatedBy.get(target) ?? []; + list.push(event.eventId); + invalidatedBy.set(target, list); + } + } catch (err) { + corruptions.push({ + kind: "invalid_reference", + eventId: event.eventId, + detail: err instanceof Error ? err.message : String(err), + }); + } + } + + return { invalidatedBy, purgedEventIds, purgedArtifactDigests, corruptions }; +} + +function applyPurge( + event: PurgeTombstoneEvent, + purgedEventIds: Set, + purgedArtifactDigests: Set, +): void { + for (const id of event.targetEventIds) purgedEventIds.add(id); + for (const digest of event.targetArtifactDigests) purgedArtifactDigests.add(digest); +} + +function validateInvalidationTargets( + event: InvalidationEvent, + all: LabEvent[], + index: number, + validEvidenceIds: Map, + purgedEventIds: Set, +): void { + for (const target of event.targetEventIds) { + if (target === event.eventId) { + throw new LabValidationError("self_target", "invalidation cannot target itself"); + } + const meta = validEvidenceIds.get(target); + if (!meta) { + // Could be unknown, or an invalidation/purge id + const earlier = all.slice(0, index).find((e) => e.eventId === target); + if (!earlier) { + throw new LabValidationError("unknown_target", `unknown target ${target}`); + } + if (earlier.eventKind === "invalidation" || earlier.eventKind === "purge_tombstone") { + throw new LabValidationError("bad_target_kind", `cannot invalidate ${earlier.eventKind}`); + } + throw new LabValidationError("unknown_target", `unknown target ${target}`); + } + if (meta.index >= index) { + throw new LabValidationError("future_target", `target ${target} is not earlier`); + } + // Purged targets: sensitive line may no longer exist — invalidation still must not + // name purge/invalidation kinds; naming a purged observation/claim id is allowed + // only if it appeared earlier as valid evidence before purge. We keep meta from + // first pass so previously-seen evidence ids remain addressable. + void purgedEventIds; + } +} + +export function isEventExcluded( + eventId: string, + index: InvalidationIndex, +): boolean { + if (index.purgedEventIds.has(eventId)) return true; + if (index.invalidatedBy.has(eventId)) return true; + return false; +} + +export function usableObservations( + events: LabEvent[], + index: InvalidationIndex, +): ObservationEvent[] { + return events.filter( + (e): e is ObservationEvent => + e.eventKind === "observation" && !isEventExcluded(e.eventId, index), + ); +} + +export function usableClaims( + events: LabEvent[], + index: InvalidationIndex, +): ClaimSnapshotEvent[] { + return events.filter( + (e): e is ClaimSnapshotEvent => + e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), + ); +} diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts new file mode 100644 index 000000000..af1e3d641 --- /dev/null +++ b/src/lab/ledger/purge.ts @@ -0,0 +1,97 @@ +import { deleteArtifactBytes, openTrustedArtifactDir } from "../artifacts/secure-fs"; +import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, PURGE_ACTIONS } from "../constants"; +import type { LabEvent, PurgeTombstoneEvent } from "../events/types"; +import { assignEventId, validateLabEvent } from "../events/validate"; +import { appendLabEvent, replayLabLedger } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import { rebuildLabProjection } from "../projection/rebuild"; +import { jcsStringify } from "../digest"; +import { readFileSync, writeFileSync } from "node:fs"; + +export interface SensitivePurgeRequest { + configDir?: string; + targetEventIds?: string[]; + targetArtifactDigests?: string[]; + purgeActions?: Array<(typeof PURGE_ACTIONS)[number]>; + recordedAt?: number; + producerVersion?: string; +} + +/** + * Exceptional sensitive-evidence purge: + * append purge_tombstone, physically remove targeted JSONL lines and artifacts, + * rebuild SQLite. Ordinary evidence is never rewritten by invalidation. + */ +export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombstoneEvent { + const paths = ensureLabDirs(req.configDir); + const targetEventIds = [...(req.targetEventIds ?? [])].sort(); + const targetArtifactDigests = [...(req.targetArtifactDigests ?? [])].sort(); + const purgeActions = [...(req.purgeActions ?? ["ledger", "sqlite", "artifact", "scratch"])].sort(); + + const tombstone = validateLabEvent( + assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "purge_tombstone" as const, + recordedAt: req.recordedAt ?? Date.now(), + producer: LAB_PRODUCER, + producerVersion: req.producerVersion ?? "2.10.2", + targetEventIds, + targetArtifactDigests, + reason: "sensitive_evidence" as const, + purgeActions, + }), + ) as PurgeTombstoneEvent; + + // Capture current events before mutation. + const replay = replayLabLedger(paths.ledgerPath); + const removeIds = new Set(targetEventIds); + + appendLabEvent(paths.ledgerPath, tombstone); + + if (purgeActions.includes("ledger")) { + const kept: LabEvent[] = []; + for (const event of replay.events) { + if (removeIds.has(event.eventId)) continue; + kept.push(event); + } + kept.push(tombstone); + const body = kept.map((e) => jcsStringify(e)).join("\n") + (kept.length ? "\n" : ""); + writeFileSync(paths.ledgerPath, body, { encoding: "utf8", mode: 0o600 }); + } + + if (purgeActions.includes("artifact")) { + const dir = openTrustedArtifactDir(paths.artifactsDir); + for (const digest of targetArtifactDigests) { + try { + deleteArtifactBytes(dir, digest); + } catch { + // already gone is acceptable for purge + } + } + // Also drop artifacts exclusively owned by removed events when listed in their refs + for (const event of replay.events) { + if (!removeIds.has(event.eventId)) continue; + if (event.eventKind === "observation") { + for (const ref of event.artifactRefs) { + try { + deleteArtifactBytes(dir, ref.digest); + } catch { + /* ignore */ + } + } + } + } + } + + if (purgeActions.includes("sqlite")) { + rebuildLabProjection(req.configDir); + } + + return tombstone; +} + +/** Test helper: read raw ledger text. */ +export function readLedgerText(configDir?: string): string { + const paths = ensureLabDirs(configDir); + return readFileSync(paths.ledgerPath, "utf8"); +} diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts new file mode 100644 index 000000000..d588fb16e --- /dev/null +++ b/src/lab/ledger/store.ts @@ -0,0 +1,146 @@ +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + readFileSync, + writeSync, +} from "node:fs"; +import { dirname } from "node:path"; +import { jcsStringify } from "../digest"; +import type { LabEvent, LedgerCorruption, ReplayResult } from "../events/types"; +import { LabValidationError, validateLabEvent } from "../events/validate"; +import { ensureLabDirs, labLedgerPath } from "../paths"; + +export interface LedgerStore { + path: string; + append(event: LabEvent): void; + replay(): ReplayResult; +} + +/** Durable append of one validated event as a single JSONL line + fsync. */ +export function appendLabEvent(ledgerPath: string, event: LabEvent): void { + const validated = validateLabEvent(event); + mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); + const line = `${jcsStringify(validated)}\n`; + const bytes = new TextEncoder().encode(line); + const fd = openSync(ledgerPath, "a", 0o600); + try { + const written = writeSync(fd, bytes); + if (written !== bytes.byteLength) { + throw new LabValidationError("short_write", "ledger append short write"); + } + fsyncSync(fd); + } finally { + closeSync(fd); + } +} + +/** + * Replay the JSONL ledger. + * Malformed or partial lines contribute no evidence and are reported as corruption. + */ +export function replayLabLedger(ledgerPath: string): ReplayResult { + const corruptions: LedgerCorruption[] = []; + const events: LabEvent[] = []; + if (!existsSync(ledgerPath)) { + return { events, corruptions, validLineCount: 0, totalLineCount: 0 }; + } + const text = readFileSync(ledgerPath, "utf8"); + if (text.length === 0) { + return { + events, + corruptions: [{ kind: "empty_ledger", detail: "ledger file is empty" }], + validLineCount: 0, + totalLineCount: 0, + }; + } + + const hasTrailingNewline = text.endsWith("\n"); + const rawLines = text.split("\n"); + // split yields a trailing empty string when file ends with \n + if (hasTrailingNewline && rawLines.length > 0 && rawLines[rawLines.length - 1] === "") { + rawLines.pop(); + } + + const seenIds = new Set(); + let totalLineCount = 0; + + for (let i = 0; i < rawLines.length; i++) { + const lineNumber = i + 1; + const line = rawLines[i]!; + totalLineCount += 1; + + // Partial final line: file does not end with newline + if (!hasTrailingNewline && i === rawLines.length - 1) { + corruptions.push({ + kind: "partial_line", + lineNumber, + detail: "partial final JSONL line (missing trailing newline)", + }); + continue; + } + + if (line.trim() === "") { + corruptions.push({ kind: "malformed_line", lineNumber, detail: "empty line" }); + continue; + } + + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + corruptions.push({ kind: "malformed_line", lineNumber, detail: "JSON parse failed" }); + continue; + } + + let event: LabEvent; + try { + event = validateLabEvent(parsed); + } catch (err) { + corruptions.push({ + kind: "invalid_event", + lineNumber, + detail: err instanceof Error ? err.message : String(err), + }); + continue; + } + + if (seenIds.has(event.eventId)) { + corruptions.push({ + kind: "duplicate_event", + lineNumber, + eventId: event.eventId, + detail: "duplicate eventId", + }); + continue; + } + seenIds.add(event.eventId); + events.push(event); + } + + return { + events, + corruptions, + validLineCount: events.length, + totalLineCount, + }; +} + +export function openLedgerStore(configDir?: string): LedgerStore { + const paths = ensureLabDirs(configDir); + return { + path: paths.ledgerPath, + append(event: LabEvent) { + appendLabEvent(paths.ledgerPath, event); + }, + replay() { + return replayLabLedger(paths.ledgerPath); + }, + }; +} + +export function defaultLedgerPath(configDir?: string): string { + return labLedgerPath(configDir); +} diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts new file mode 100644 index 000000000..f474cf8db --- /dev/null +++ b/src/lab/observe/from-conformance.ts @@ -0,0 +1,258 @@ +/** + * Persistence seam: transform CL-01 conformance results into valid observation events. + * Deterministic harness execution stays separate from ledger persistence. + */ +import { createHash } from "node:crypto"; +import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + type ObservationOutcome, +} from "../constants"; +import { + jcsStringify, + scenarioManifestDigest, + subjectIdForSubject, + suiteManifestDigest, +} from "../digest"; +import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; +import { assignEventId } from "../events/validate"; +import { appendLabEvent } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import type { CaseAuthority, CaseRecord, ScenarioRunResult } from "../conformance/types"; +import { expandScenario } from "../conformance/manifest"; +import { fixtureDigest } from "../conformance/digest"; + +const PACKAGE_VERSION = "2.10.2"; +const COMPAT_VERSION = "protocol-v1"; + +export interface PersistConformanceOptions { + configDir?: string; + recordedAt?: number; + producerVersion?: string; + artifactStore?: ArtifactStore; +} + +export interface PersistedConformanceObservation { + event: ObservationEvent; + ledgerPath: string; +} + +function behaviorFingerprintForCase(caseRecord: CaseRecord): string { + const values = { + schemaVersion: 1, + resolverVersion: 1, + values: { + "wire.adapter": { + source: "lab_forced", + value: caseRecord.requirements.upstreamProtocols[0] ?? "unknown", + }, + "wire.upstreamProtocol": { + source: "lab_forced", + value: caseRecord.requirements.upstreamProtocols[0] ?? "unknown", + }, + "lab.suite": { source: "lab_forced", value: caseRecord.suite }, + "lab.scenario": { source: "lab_forced", value: caseRecord.id }, + }, + }; + return createHash("sha256").update(jcsStringify(values)).digest("hex"); +} + +function protocolSubject(caseRecord: CaseRecord): ProtocolSubjectV1 { + const inbound = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const surface = caseRecord.requirements.surfaces[0] ?? "responses-http"; + return { + subjectSchemaVersion: 1, + subjectKind: "protocol", + opencodexCompatibilityVersion: COMPAT_VERSION, + effectiveAdapter: upstreamAdapter(upstream), + inboundProtocol: inbound, + upstreamProtocol: upstream, + surface, + behaviorFingerprint: behaviorFingerprintForCase(caseRecord), + }; +} + +function upstreamAdapter(protocol: string): string { + switch (protocol) { + case "openai-responses": + return "openai-responses"; + case "anthropic-messages": + return "anthropic"; + default: + return "openai-chat"; + } +} + +function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { + if (result.passed) return "pass"; + if (result.classification === "timeout" || result.classification === "budget_exhausted") { + return "blocked"; + } + if (result.classification === "inconclusive" || result.classification === "harness_failure") { + return "inconclusive"; + } + return "fail"; +} + +function expandSuiteManifest(caseRecord: CaseRecord, authority: CaseAuthority): Record { + const defaults = authority.manifestDefaults; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + scenarioIds: authority.cases.filter((c) => c.suite === caseRecord.suite).map((c) => c.id).sort(), + }; +} + +/** + * Build a valid protocol_conformance observation from one CL-01 scenario result. + * Does not append; use persistConformanceResult for ledger write. + */ +export function observationFromConformanceResult( + result: ScenarioRunResult, + caseRecord: CaseRecord, + authority: CaseAuthority, + opts: PersistConformanceOptions = {}, +): { event: ObservationEvent; artifacts: ReturnType[] } { + const paths = ensureLabDirs(opts.configDir); + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + const recordedAt = opts.recordedAt ?? Date.now(); + const startedAt = recordedAt - 1; + const completedAt = recordedAt; + + const expandedScenario = expandScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expandedScenario); + const suiteExpanded = expandSuiteManifest(caseRecord, authority); + const suiteDigest = suiteManifestDigest(suiteExpanded); + + const fixtureDigests: string[] = []; + const artifacts: ReturnType[] = []; + + const putFixture = (fixture: CaseRecord["fixture"]) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + const digest = fixtureDigest(bytes); + fixtureDigests.push(digest); + artifacts.push( + store.put({ + artifactClass: "fixture", + payload: bytes, + expectedDigest: digest, + mediaType: fixture.mediaType, + }), + ); + }; + putFixture(caseRecord.fixture); + if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); + + artifacts.push( + store.put({ + artifactClass: "scenario_manifest", + payload: expandedScenario, + expectedDigest: scenarioDigest, + }), + ); + artifacts.push( + store.put({ + artifactClass: "suite_manifest", + payload: suiteExpanded, + expectedDigest: suiteDigest, + }), + ); + + const assertionReport = store.put({ + artifactClass: "assertion_report", + payload: { + scenarioId: result.scenarioId, + passed: result.passed, + classification: result.classification, + assertions: result.assertionResults.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: a.passed, + observedSummary: a.observedSummary, + reason: a.reason, + })), + }, + }); + artifacts.push(assertionReport); + + const subject = protocolSubject(caseRecord); + const subjectId = subjectIdForSubject(subject); + const outcome = outcomeFromResult(result); + + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt, + producer: LAB_PRODUCER, + producerVersion: opts.producerVersion ?? PACKAGE_VERSION, + evidenceLayer: "protocol_conformance" as const, + scenarioId: caseRecord.id, + scenarioVersion: String(authority.manifestDefaults.version), + scenarioManifestDigest: scenarioDigest, + suiteId: caseRecord.suite, + suiteVersion: String(authority.manifestDefaults.suiteVersion), + suiteManifestDigest: suiteDigest, + fixtureDigests, + subject, + subjectId, + startedAt, + completedAt, + executionMode: "fixture" as const, + attempt: 1, + limits: { ...(authority.manifestDefaults.executionLimits as Record) }, + outcome, + assertions: result.assertionResults.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: a.passed, + expectedSummary: "see_assertion_report", + observedSummary: a.observedSummary.slice(0, 512), + ...(a.reason ? { reason: a.reason } : {}), + })), + environment: { + runtime: { + platform: process.platform, + arch: process.arch, + }, + }, + artifactRefs: artifacts, + ...(result.passed + ? {} + : { + failure: { + class: result.classification, + code: result.secondaryCode ?? result.classification, + retryable: false, + attribution: + result.classification === "harness_failure" + ? ("harness" as const) + : ("opencodex" as const), + }, + }), + }; + + const event = assignEventId(eventWithoutId) as ObservationEvent; + return { event, artifacts }; +} + +/** Transform → validate → append one CL-01 result into the canonical JSONL ledger. */ +export function persistConformanceResult( + result: ScenarioRunResult, + caseRecord: CaseRecord, + authority: CaseAuthority, + opts: PersistConformanceOptions = {}, +): PersistedConformanceObservation { + const paths = ensureLabDirs(opts.configDir); + const { event } = observationFromConformanceResult(result, caseRecord, authority, { + ...opts, + artifactStore: opts.artifactStore ?? createArtifactStore(paths.artifactsDir), + }); + appendLabEvent(paths.ledgerPath, event); + return { event, ledgerPath: paths.ledgerPath }; +} diff --git a/src/lab/paths.ts b/src/lab/paths.ts new file mode 100644 index 000000000..a40befc5d --- /dev/null +++ b/src/lab/paths.ts @@ -0,0 +1,39 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { getConfigDir } from "../config"; + +/** Canonical Compatibility Lab state root under the OpenCodex config dir. */ +export function labRoot(configDir = getConfigDir()): string { + return join(configDir, "lab"); +} + +export function labLedgerPath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "compatibility.jsonl"); +} + +export function labSqlitePath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "compatibility.sqlite"); +} + +export function labArtifactsDir(configDir = getConfigDir()): string { + return join(labRoot(configDir), "artifacts"); +} + +/** Ensure lab directories exist with restrictive permissions where the platform allows. */ +export function ensureLabDirs(configDir = getConfigDir()): { + root: string; + ledgerPath: string; + sqlitePath: string; + artifactsDir: string; +} { + const root = labRoot(configDir); + const artifactsDir = labArtifactsDir(configDir); + mkdirSync(root, { recursive: true, mode: 0o700 }); + mkdirSync(artifactsDir, { recursive: true, mode: 0o700 }); + return { + root, + ledgerPath: labLedgerPath(configDir), + sqlitePath: labSqlitePath(configDir), + artifactsDir, + }; +} diff --git a/src/lab/projection/rebuild.ts b/src/lab/projection/rebuild.ts new file mode 100644 index 000000000..57f02d5f9 --- /dev/null +++ b/src/lab/projection/rebuild.ts @@ -0,0 +1,378 @@ +import { Database } from "bun:sqlite"; +import { existsSync, unlinkSync } from "node:fs"; +import { createArtifactStore } from "../artifacts/store"; +import { ArtifactFsError } from "../artifacts/secure-fs"; +import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; +import { jcsStringify } from "../digest"; +import type { LabEvent, LedgerCorruption } from "../events/types"; +import { loadClaimSourceManifest } from "../artifacts/store"; +import { buildInvalidationIndex, isEventExcluded } from "../ledger/invalidation"; +import { replayLabLedger } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import { LAB_SQLITE_DDL, LAB_SQLITE_SCHEMA_VERSION } from "./schema"; +import { + excludeEventIds, + projectVerdicts, + projectionKeyString, + resolveClaimStates, +} from "./verdicts"; + +export interface RebuildResult { + events: number; + verdicts: number; + corruptions: LedgerCorruption[]; + sqlitePath: string; +} + +function wipeSqlite(path: string): void { + // Prefer deleting the file; on Windows fall back to leaving it for truncate recreate. + for (const candidate of [path, `${path}-wal`, `${path}-shm`]) { + for (let attempt = 0; attempt < 8; attempt++) { + try { + if (existsSync(candidate)) unlinkSync(candidate); + break; + } catch (err) { + const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; + if (code !== "EBUSY" && code !== "EPERM") throw err; + Bun.sleepSync(20 * (attempt + 1)); + } + } + } +} + +function resetProjectionSchema(db: Database): void { + db.exec("PRAGMA foreign_keys=OFF;"); + db.exec(` + DROP TABLE IF EXISTS verdicts; + DROP TABLE IF EXISTS corruption; + DROP TABLE IF EXISTS artifacts; + DROP TABLE IF EXISTS purges; + DROP TABLE IF EXISTS invalidations; + DROP TABLE IF EXISTS claims; + DROP TABLE IF EXISTS observations; + DROP TABLE IF EXISTS subjects; + DROP TABLE IF EXISTS events; + DROP TABLE IF EXISTS schema_meta; + `); + db.exec("PRAGMA foreign_keys=ON;"); + db.exec(LAB_SQLITE_DDL); +} + +/** + * Deterministic rebuild: + * delete compatibility.sqlite → replay JSONL → validate artifacts → project. + */ +export function rebuildLabProjection(configDir?: string): RebuildResult { + const paths = ensureLabDirs(configDir); + wipeSqlite(paths.sqlitePath); + + const replay = replayLabLedger(paths.ledgerPath); + const corruptions: LedgerCorruption[] = [...replay.corruptions]; + const index = buildInvalidationIndex(replay.events); + corruptions.push(...index.corruptions); + + const artifactStore = createArtifactStore(paths.artifactsDir); + validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); + + const db = new Database(paths.sqlitePath); + try { + db.exec("PRAGMA journal_mode=DELETE;"); + resetProjectionSchema(db); + db.prepare( + "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", + ).run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION)); + db.prepare( + "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", + ).run("projection_spec_version", LAB_PROJECTION_SPEC_VERSION); + db.prepare( + "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", + ).run("built_at_ms", String(Date.now())); + + const insertCorruption = db.prepare( + "INSERT INTO corruption(kind, line_number, event_id, detail) VALUES (?, ?, ?, ?)", + ); + for (const c of corruptions) { + insertCorruption.run(c.kind, c.lineNumber ?? null, c.eventId ?? null, c.detail); + } + + const insertEvent = db.prepare( + `INSERT INTO events(event_id, event_kind, recorded_at, producer, producer_version, payload_json, excluded, exclusion_reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ); + const insertSubject = db.prepare( + `INSERT OR IGNORE INTO subjects(subject_id, subject_kind, subject_json) VALUES (?, ?, ?)`, + ); + const insertObs = db.prepare( + `INSERT INTO observations( + event_id, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, + scenario_id, scenario_version, scenario_manifest_digest, outcome, completed_at, execution_mode + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + const insertClaim = db.prepare( + `INSERT INTO claims( + event_id, subject_id, capability, polarity, source_manifest_digest, + effective_at, recorded_at, supersedes_json, current, usable + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + const insertInv = db.prepare( + `INSERT INTO invalidations(event_id, reason, targets_json, recorded_at, applied) VALUES (?, ?, ?, ?, ?)`, + ); + const insertPurge = db.prepare( + `INSERT INTO purges(event_id, target_event_ids_json, target_artifact_digests_json, purge_actions_json, recorded_at) + VALUES (?, ?, ?, ?, ?)`, + ); + const insertArtifact = db.prepare( + `INSERT OR REPLACE INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error) + VALUES (?, ?, ?, ?, ?, ?)`, + ); + + const excluded = excludeEventIds(index); + const usableClaimEvents = replay.events.filter( + (e) => e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), + ); + const claimStates = resolveClaimStates( + usableClaimEvents.filter((e) => e.eventKind === "claim_snapshot") as never, + ); + + for (const event of replay.events) { + const isExcluded = excluded.has(event.eventId); + let exclusionReason: string | null = null; + if (index.purgedEventIds.has(event.eventId)) exclusionReason = "purged"; + else if (index.invalidatedBy.has(event.eventId)) exclusionReason = "invalidated"; + + insertEvent.run( + event.eventId, + event.eventKind, + event.recordedAt, + event.producer, + event.producerVersion, + jcsStringify(event), + isExcluded ? 1 : 0, + exclusionReason, + ); + + if (event.eventKind === "observation") { + insertSubject.run(event.subjectId, event.subject.subjectKind, jcsStringify(event.subject)); + if (!isExcluded) { + insertObs.run( + event.eventId, + event.subjectId, + event.evidenceLayer, + event.suiteId, + event.suiteVersion, + event.suiteManifestDigest, + event.scenarioId, + event.scenarioVersion, + event.scenarioManifestDigest, + event.outcome, + event.completedAt, + event.executionMode, + ); + } + for (const ref of event.artifactRefs) { + const purged = index.purgedArtifactDigests.has(ref.digest); + insertArtifact.run( + ref.digest, + ref.artifactClass, + ref.mediaType, + ref.byteCount, + purged ? "purged_unavailable" : "present", + null, + ); + } + } else if (event.eventKind === "claim_snapshot") { + insertSubject.run(event.subjectId, event.subject.subjectKind, jcsStringify(event.subject)); + const key = `${event.subjectId}|${event.capability}`; + const state = claimStates.states.get(key); + const current = state?.current?.eventId === event.eventId ? 1 : 0; + let usable = !isExcluded && !state?.corruption ? 1 : 0; + + if (!isExcluded) { + const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { + subjectId: event.subjectId, + capability: event.capability, + }); + if (loaded.corruption) { + usable = 0; + corruptions.push({ + kind: "claim_corruption", + eventId: event.eventId, + detail: loaded.corruption, + }); + insertCorruption.run("claim_corruption", null, event.eventId, loaded.corruption); + insertArtifact.run( + event.sourceManifestDigest, + "claim_source_manifest", + "application/json", + null, + "corrupt", + loaded.corruption, + ); + } else { + insertArtifact.run( + event.sourceManifestDigest, + "claim_source_manifest", + "application/json", + null, + "present", + null, + ); + } + } + + insertClaim.run( + event.eventId, + event.subjectId, + event.capability, + event.polarity, + event.sourceManifestDigest, + event.effectiveAt, + event.recordedAt, + jcsStringify(event.supersedes), + current, + usable, + ); + } else if (event.eventKind === "invalidation") { + const applied = !corruptions.some((c) => c.eventId === event.eventId && c.kind === "invalid_reference"); + insertInv.run( + event.eventId, + event.reason, + jcsStringify(event.targetEventIds), + event.recordedAt, + applied ? 1 : 0, + ); + } else if (event.eventKind === "purge_tombstone") { + insertPurge.run( + event.eventId, + jcsStringify(event.targetEventIds), + jcsStringify(event.targetArtifactDigests), + jcsStringify(event.purgeActions), + event.recordedAt, + ); + for (const digest of event.targetArtifactDigests) { + insertArtifact.run(digest, null, null, null, "purged_unavailable", null); + } + } + } + + const { verdicts, corruptions: verdictCorruptions } = projectVerdicts(replay.events, { index }); + for (const c of verdictCorruptions) { + if (!corruptions.some((x) => x.detail === c.detail && x.eventId === c.eventId)) { + corruptions.push(c); + insertCorruption.run(c.kind, c.lineNumber ?? null, c.eventId ?? null, c.detail); + } + } + + const insertVerdict = db.prepare( + `INSERT INTO verdicts( + projection_key, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, + projection_spec_version, verdict, as_of, scenario_manifest_digests_json, claim_source_digest, + contributing_event_ids_json, contradicting_event_ids_json, notes_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + + for (const v of verdicts) { + // Purged evidence must not contribute to cached verdicts. + if (v.contributingEventIds.every((id) => index.purgedEventIds.has(id))) continue; + if (v.contributingEventIds.some((id) => index.purgedEventIds.has(id) || index.invalidatedBy.has(id))) { + // Drop any verdict that still lists excluded evidence as contributing. + const remaining = v.contributingEventIds.filter( + (id) => !index.purgedEventIds.has(id) && !index.invalidatedBy.has(id), + ); + if (remaining.length === 0) continue; + } + insertVerdict.run( + projectionKeyString(v.key), + v.key.subjectId, + v.key.evidenceLayer, + v.key.suiteId, + v.key.suiteVersion, + v.key.suiteManifestDigest, + v.key.projectionSpecVersion, + v.verdict, + v.asOf, + jcsStringify(v.scenarioManifestDigests), + v.claimSourceDigest ?? null, + jcsStringify(v.contributingEventIds), + jcsStringify(v.contradictingEventIds), + jcsStringify(v.notes), + ); + } + + return { + events: replay.events.length, + verdicts: verdicts.length, + corruptions, + sqlitePath: paths.sqlitePath, + }; + } finally { + db.close(); + } +} + +function validateRequiredArtifacts( + events: LabEvent[], + index: ReturnType, + artifactStore: ReturnType, + corruptions: LedgerCorruption[], +): void { + for (const event of events) { + if (isEventExcluded(event.eventId, index)) continue; + if (event.eventKind === "observation") { + const required = [ + event.scenarioManifestDigest, + event.suiteManifestDigest, + ...event.fixtureDigests, + ]; + for (const digest of required) { + if (index.purgedArtifactDigests.has(digest)) { + corruptions.push({ + kind: "missing_artifact", + eventId: event.eventId, + detail: `required artifact purged: ${digest}`, + }); + continue; + } + try { + artifactStore.get(digest); + } catch (err) { + corruptions.push({ + kind: err instanceof ArtifactFsError && err.message.includes("mismatch") + ? "artifact_mismatch" + : "missing_artifact", + eventId: event.eventId, + detail: err instanceof Error ? err.message : String(err), + }); + } + } + } + } +} + +/** Snapshot derived verdict rows for rebuild-determinism tests (excludes asOf wall clock). */ +export function readVerdictSnapshot(sqlitePath: string): unknown[] { + const db = new Database(sqlitePath, { readonly: true }); + try { + const rows = db + .query( + `SELECT projection_key, subject_id, evidence_layer, suite_id, suite_version, + suite_manifest_digest, projection_spec_version, verdict, + scenario_manifest_digests_json, claim_source_digest, + contributing_event_ids_json, contradicting_event_ids_json, notes_json + FROM verdicts ORDER BY projection_key`, + ) + .all(); + return rows; + } finally { + db.close(); + } +} + +export function readCorruptionRows(sqlitePath: string): unknown[] { + const db = new Database(sqlitePath, { readonly: true }); + try { + return db.query(`SELECT kind, line_number, event_id, detail FROM corruption ORDER BY id`).all(); + } finally { + db.close(); + } +} diff --git a/src/lab/projection/schema.ts b/src/lab/projection/schema.ts new file mode 100644 index 000000000..344b6595d --- /dev/null +++ b/src/lab/projection/schema.ts @@ -0,0 +1,122 @@ +/** + * Disposable SQLite projection schema for Compatibility Lab (CL-02). + * Not canonical storage — rebuildable from JSONL + content-addressed artifacts. + */ +export const LAB_SQLITE_SCHEMA_VERSION = 1; + +export const LAB_SQLITE_DDL = ` +CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS events ( + event_id TEXT PRIMARY KEY, + event_kind TEXT NOT NULL, + recorded_at INTEGER NOT NULL, + producer TEXT NOT NULL, + producer_version TEXT NOT NULL, + payload_json TEXT NOT NULL, + excluded INTEGER NOT NULL DEFAULT 0, + exclusion_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_events_kind ON events(event_kind, recorded_at); +CREATE INDEX IF NOT EXISTS idx_events_excluded ON events(excluded); + +CREATE TABLE IF NOT EXISTS subjects ( + subject_id TEXT PRIMARY KEY, + subject_kind TEXT NOT NULL, + subject_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS observations ( + event_id TEXT PRIMARY KEY, + subject_id TEXT NOT NULL, + evidence_layer TEXT NOT NULL, + suite_id TEXT NOT NULL, + suite_version TEXT NOT NULL, + suite_manifest_digest TEXT NOT NULL, + scenario_id TEXT NOT NULL, + scenario_version TEXT NOT NULL, + scenario_manifest_digest TEXT NOT NULL, + outcome TEXT NOT NULL, + completed_at INTEGER NOT NULL, + execution_mode TEXT NOT NULL, + FOREIGN KEY(event_id) REFERENCES events(event_id) +); + +CREATE INDEX IF NOT EXISTS idx_observations_proj ON observations( + subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, completed_at +); + +CREATE TABLE IF NOT EXISTS claims ( + event_id TEXT PRIMARY KEY, + subject_id TEXT NOT NULL, + capability TEXT NOT NULL, + polarity TEXT NOT NULL, + source_manifest_digest TEXT NOT NULL, + effective_at INTEGER NOT NULL, + recorded_at INTEGER NOT NULL, + supersedes_json TEXT NOT NULL, + current INTEGER NOT NULL DEFAULT 0, + usable INTEGER NOT NULL DEFAULT 1, + FOREIGN KEY(event_id) REFERENCES events(event_id) +); + +CREATE INDEX IF NOT EXISTS idx_claims_key ON claims(subject_id, capability, effective_at, event_id); + +CREATE TABLE IF NOT EXISTS invalidations ( + event_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + targets_json TEXT NOT NULL, + recorded_at INTEGER NOT NULL, + applied INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY(event_id) REFERENCES events(event_id) +); + +CREATE TABLE IF NOT EXISTS purges ( + event_id TEXT PRIMARY KEY, + target_event_ids_json TEXT NOT NULL, + target_artifact_digests_json TEXT NOT NULL, + purge_actions_json TEXT NOT NULL, + recorded_at INTEGER NOT NULL, + FOREIGN KEY(event_id) REFERENCES events(event_id) +); + +CREATE TABLE IF NOT EXISTS artifacts ( + digest TEXT PRIMARY KEY, + artifact_class TEXT, + media_type TEXT, + byte_count INTEGER, + status TEXT NOT NULL, + last_error TEXT +); + +CREATE TABLE IF NOT EXISTS verdicts ( + projection_key TEXT PRIMARY KEY, + subject_id TEXT NOT NULL, + evidence_layer TEXT NOT NULL, + suite_id TEXT NOT NULL, + suite_version TEXT NOT NULL, + suite_manifest_digest TEXT NOT NULL, + projection_spec_version TEXT NOT NULL, + verdict TEXT NOT NULL, + as_of INTEGER NOT NULL, + scenario_manifest_digests_json TEXT NOT NULL, + claim_source_digest TEXT, + contributing_event_ids_json TEXT NOT NULL, + contradicting_event_ids_json TEXT NOT NULL, + notes_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_verdicts_subject ON verdicts(subject_id, evidence_layer, suite_id); + +CREATE TABLE IF NOT EXISTS corruption ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + line_number INTEGER, + event_id TEXT, + detail TEXT NOT NULL +); +`; diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts new file mode 100644 index 000000000..c7054ebf3 --- /dev/null +++ b/src/lab/projection/verdicts.ts @@ -0,0 +1,301 @@ +import type { CompatibilityVerdict } from "../constants"; +import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; +import type { + ClaimSnapshotEvent, + LabEvent, + LedgerCorruption, + ObservationEvent, +} from "../events/types"; +import { + buildInvalidationIndex, + isEventExcluded, + usableClaims, + usableObservations, + type InvalidationIndex, +} from "../ledger/invalidation"; + +export interface ProjectionKey { + subjectId: string; + evidenceLayer: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + projectionSpecVersion: string; +} + +export function projectionKeyString(key: ProjectionKey): string { + return [ + key.subjectId, + key.evidenceLayer, + key.suiteId, + key.suiteVersion, + key.suiteManifestDigest, + key.projectionSpecVersion, + ].join("|"); +} + +export interface DerivedVerdict { + key: ProjectionKey; + verdict: CompatibilityVerdict; + asOf: number; + scenarioManifestDigests: string[]; + claimSourceDigest?: string; + contributingEventIds: string[]; + contradictingEventIds: string[]; + notes: string[]; +} + +export interface ClaimState { + key: string; + current: ClaimSnapshotEvent | null; + corruption?: string; +} + +/** + * Resolve current claims after purge/invalidation and supersession. + * Multiple unsuperseded claims, missing predecessors, cross-key supersession, or cycles → UNKNOWN + corruption. + */ +export function resolveClaimStates(claims: ClaimSnapshotEvent[]): { + states: Map; + corruptions: LedgerCorruption[]; +} { + const byKey = new Map(); + for (const claim of claims) { + const key = `${claim.subjectId}|${claim.capability}`; + const list = byKey.get(key) ?? []; + list.push(claim); + byKey.set(key, list); + } + + const states = new Map(); + const corruptions: LedgerCorruption[] = []; + + for (const [key, list] of byKey) { + const sorted = [...list].sort((a, b) => { + if (a.effectiveAt !== b.effectiveAt) return a.effectiveAt - b.effectiveAt; + if (a.recordedAt !== b.recordedAt) return a.recordedAt - b.recordedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }); + + const superseded = new Set(); + const byId = new Map(sorted.map((c) => [c.eventId, c])); + + for (const claim of sorted) { + for (const pred of claim.supersedes) { + const prev = byId.get(pred); + if (!prev) { + corruptions.push({ + kind: "claim_corruption", + eventId: claim.eventId, + detail: `missing supersedes predecessor ${pred}`, + }); + states.set(key, { key, current: null, corruption: "missing predecessor" }); + continue; + } + if (prev.subjectId !== claim.subjectId || prev.capability !== claim.capability) { + corruptions.push({ + kind: "claim_corruption", + eventId: claim.eventId, + detail: "cross-key supersession", + }); + states.set(key, { key, current: null, corruption: "cross-key supersession" }); + continue; + } + superseded.add(pred); + } + } + + if (states.get(key)?.corruption) continue; + + const current = sorted.filter((c) => !superseded.has(c.eventId)); + if (current.length > 1) { + corruptions.push({ + kind: "claim_corruption", + detail: `multiple unsuperseded claims for ${key}`, + }); + states.set(key, { key, current: null, corruption: "conflicting current claims" }); + continue; + } + states.set(key, { key, current: current[0] ?? null }); + } + + return { states, corruptions }; +} + +/** + * CL-02 verdict projection primitives. + * + * Full live-route/task coverage algorithms belong to later phases. This implements + * the reusable precedence/freshness/invalidation semantics for protocol_conformance + * observations and claim snapshots needed for persistence tests. + */ +export function projectVerdicts( + events: LabEvent[], + opts: { asOf?: number; index?: InvalidationIndex } = {}, +): { verdicts: DerivedVerdict[]; corruptions: LedgerCorruption[]; index: InvalidationIndex } { + const index = opts.index ?? buildInvalidationIndex(events); + const corruptions = [...index.corruptions]; + const asOf = + opts.asOf ?? + events.reduce((max, event) => { + if (event.eventKind === "observation") return Math.max(max, event.completedAt, event.recordedAt); + if (event.eventKind === "claim_snapshot") return Math.max(max, event.effectiveAt, event.recordedAt); + return Math.max(max, event.recordedAt); + }, 0); + + const observations = usableObservations(events, index).filter((o) => o.completedAt <= asOf); + const claims = usableClaims(events, index).filter((c) => c.effectiveAt <= asOf); + const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims); + corruptions.push(...claimCorruptions); + + const groups = new Map(); + for (const obs of observations) { + const key: ProjectionKey = { + subjectId: obs.subjectId, + evidenceLayer: obs.evidenceLayer, + suiteId: obs.suiteId, + suiteVersion: obs.suiteVersion, + suiteManifestDigest: obs.suiteManifestDigest, + projectionSpecVersion: LAB_PROJECTION_SPEC_VERSION, + }; + const ks = projectionKeyString(key); + const list = groups.get(ks) ?? []; + list.push(obs); + groups.set(ks, list); + } + + const verdicts: DerivedVerdict[] = []; + + for (const [, list] of groups) { + const sample = list[0]!; + const key: ProjectionKey = { + subjectId: sample.subjectId, + evidenceLayer: sample.evidenceLayer, + suiteId: sample.suiteId, + suiteVersion: sample.suiteVersion, + suiteManifestDigest: sample.suiteManifestDigest, + projectionSpecVersion: LAB_PROJECTION_SPEC_VERSION, + }; + const ordered = [...list].sort((a, b) => { + if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }); + verdicts.push(projectObservationGroup(key, ordered, asOf)); + } + + // Claim-only keys (live_route_compatibility) when no executable observations exist. + for (const [, state] of claimStates) { + if (!state.current || state.current.polarity !== "supported") continue; + const claim = state.current; + const key: ProjectionKey = { + subjectId: claim.subjectId, + evidenceLayer: "live_route_compatibility", + suiteId: "_claims", + suiteVersion: "1", + suiteManifestDigest: claim.sourceManifestDigest, + projectionSpecVersion: LAB_PROJECTION_SPEC_VERSION, + }; + const ks = projectionKeyString(key); + if (verdicts.some((v) => projectionKeyString(v.key) === ks)) continue; + // Claims cannot produce PROBED/VERIFIED. + verdicts.push({ + key, + verdict: "CLAIMED", + asOf, + scenarioManifestDigests: [], + claimSourceDigest: claim.sourceManifestDigest, + contributingEventIds: [claim.eventId], + contradictingEventIds: [], + notes: ["claim_snapshot"], + }); + } + + return { verdicts, corruptions, index }; +} + +function projectObservationGroup( + key: ProjectionKey, + ordered: ObservationEvent[], + asOf: number, +): DerivedVerdict { + const contributing: string[] = []; + const contradicting: string[] = []; + const digests = new Set(); + const notes: string[] = []; + + // Conservative protocol projection for CL-02: + // - all required-role passes for covered scenarios → VERIFIED when every obs passes + // - mix of pass/fail → DEGRADED (newer required failure prevents VERIFIED) + // - only passes but incomplete suite metadata → PROBED + // - blocked → BLOCKED if no conclusive result + // Protocol claims are forbidden — never CLAIMED here. + + let sawPass = false; + let sawFail = false; + let sawBlocked = false; + let sawInconclusive = false; + + for (const obs of ordered) { + digests.add(obs.scenarioManifestDigest); + contributing.push(obs.eventId); + if (obs.outcome === "pass") sawPass = true; + else if (obs.outcome === "fail") { + sawFail = true; + contradicting.push(obs.eventId); + } else if (obs.outcome === "blocked") sawBlocked = true; + else if (obs.outcome === "inconclusive") sawInconclusive = true; + else { + const _never: never = obs.outcome; + void _never; + } + } + + let verdict: CompatibilityVerdict = "UNKNOWN"; + if (key.evidenceLayer !== "protocol_conformance" && key.evidenceLayer !== "live_route_compatibility" && key.evidenceLayer !== "task_effectiveness") { + verdict = "UNKNOWN"; + } else if (sawFail && sawPass) { + // Newer failure prevents VERIFIED; remain DEGRADED until repair coverage. + verdict = "DEGRADED"; + notes.push("contradiction_conservative_v1"); + } else if (sawFail) { + const last = ordered[ordered.length - 1]!; + if (last.failure?.class === "capability_failure" && last.expectedFailure) { + verdict = "UNSUPPORTED"; + notes.push("capability_absence_control"); + } else { + verdict = "DEGRADED"; + } + } else if (sawPass && !sawInconclusive && !sawBlocked) { + // Full verification rule evaluation is suite-manifest driven in later phases. + // CL-02 treats an all-pass observation set for this projection key as VERIFIED + // only when every observation is protocol_conformance fixture mode; otherwise PROBED. + const allFixtureProtocol = ordered.every( + (o) => o.evidenceLayer === "protocol_conformance" && o.executionMode === "fixture", + ); + verdict = allFixtureProtocol ? "VERIFIED" : "PROBED"; + } else if (sawPass) { + verdict = "PROBED"; + } else if (sawBlocked) { + verdict = "BLOCKED"; + } else { + verdict = "UNKNOWN"; + } + + return { + key, + verdict, + asOf, + scenarioManifestDigests: [...digests].sort(), + contributingEventIds: contributing, + contradictingEventIds: contradicting, + notes, + }; +} + +export function excludeEventIds(index: InvalidationIndex): Set { + const out = new Set([...index.purgedEventIds]); + for (const id of index.invalidatedBy.keys()) out.add(id); + return out; +} + +export { isEventExcluded }; diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts new file mode 100644 index 000000000..3d4cdd62d --- /dev/null +++ b/tests/lab-evidence-ledger.test.ts @@ -0,0 +1,632 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { chmodSync, existsSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + appendLabEvent, + assignEventId, + buildInvalidationIndex, + claimSourceManifestDigest, + createArtifactStore, + eventIdForPayload, + isSha256Hex, + jcsStringify, + observationFromConformanceResult, + openTrustedArtifactDir, + persistConformanceResult, + projectVerdicts, + purgeSensitiveEvidence, + readVerdictSnapshot, + rebuildLabProjection, + replayLabLedger, + resolveClaimStates, + subjectIdForSubject, + validateClaimSourceManifest, + validateLabEvent, + validateSortedUniqueHexIds, + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PROJECTION_SPEC_VERSION, +} from "../src/lab"; +import { ArtifactFsError, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; +import { discoverScenarios, loadCaseAuthority } from "../src/lab/conformance/manifest"; +import { runScenario } from "../src/lab/conformance/executor"; +import type { ClaimSnapshotEvent, ObservationEvent, ProtocolSubjectV1 } from "../src/lab/events/types"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +beforeEach(() => { + // OPENCODEX_HOME isolation +}); + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + } + delete process.env.OPENCODEX_HOME; +}); + +function withHome(fn: (home: string) => T): T { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + return fn(home); +} + +function protocolSubject(seed = "a"): ProtocolSubjectV1 { + return { + subjectSchemaVersion: 1, + subjectKind: "protocol", + opencodexCompatibilityVersion: "protocol-v1", + effectiveAdapter: "openai-chat", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-chat", + surface: "responses-http", + behaviorFingerprint: createHashHex(seed), + }; +} + +function createHashHex(s: string): string { + return Bun.CryptoHasher.hash("sha256", s, "hex"); +} + +function baseObservation(overrides: Partial = {}): ObservationEvent { + const subject = protocolSubject(overrides.scenarioId ?? "s1"); + const subjectId = subjectIdForSubject(subject); + const fixtureDigest = createHashHex("fixture"); + const scenarioManifestDigest = createHashHex("scenario"); + const suiteManifestDigest = createHashHex("suite"); + const event = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt: 1_700_000_000_000, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + evidenceLayer: "protocol_conformance" as const, + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest, + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest, + fixtureDigests: [fixtureDigest], + subject, + subjectId, + startedAt: 1_700_000_000_000, + completedAt: 1_700_000_000_100, + executionMode: "fixture" as const, + attempt: 1, + limits: { maxWallClockMs: 1000 }, + outcome: "pass" as const, + assertions: [ + { + id: "a1", + operator: "equals", + required: true, + passed: true, + expectedSummary: "ok", + observedSummary: "ok", + }, + ], + environment: { runtime: { platform: "test" } }, + artifactRefs: [ + { + digest: scenarioManifestDigest, + mediaType: "application/json", + byteCount: 2, + redactionPolicy: "contract_canonical_v1", + relativePath: `${scenarioManifestDigest}.bin`, + artifactClass: "scenario_manifest", + }, + { + digest: suiteManifestDigest, + mediaType: "application/json", + byteCount: 2, + redactionPolicy: "contract_canonical_v1", + relativePath: `${suiteManifestDigest}.bin`, + artifactClass: "suite_manifest", + }, + { + digest: fixtureDigest, + mediaType: "application/json", + byteCount: 2, + redactionPolicy: "contract_canonical_v1", + relativePath: `${fixtureDigest}.bin`, + artifactClass: "fixture", + }, + ], + ...overrides, + }); + return event as ObservationEvent; +} + +describe("CL-02 JCS and IDs", () => { + test("eventId and subjectId are domain-separated deterministic digests", () => { + const subject = protocolSubject("x"); + const subjectId = subjectIdForSubject(subject); + expect(isSha256Hex(subjectId)).toBe(true); + expect(subjectId).toBe(subjectIdForSubject(subject)); + + const payload = { + schemaVersion: 1, + eventKind: "invalidation", + recordedAt: 1, + producer: "p", + producerVersion: "1", + targetEventIds: [createHashHex("t")], + reason: "harness_defect", + }; + const id1 = eventIdForPayload(payload); + const id2 = eventIdForPayload(payload); + expect(id1).toBe(id2); + expect(isSha256Hex(id1)).toBe(true); + // Domain separation: different domain changes digest + expect(id1).not.toBe(subjectId); + }); + + test("JCS key order is deterministic", () => { + expect(jcsStringify({ b: 1, a: 2 })).toBe('{"a":2,"b":1}'); + }); +}); + +describe("CL-02 ledger append/replay", () => { + test("append and replay round-trip", () => { + withHome((home) => { + const event = baseObservation(); + // Store required artifacts as opaque named digests for projection + const store = createArtifactStore(join(home, "lab", "artifacts")); + for (const ref of event.artifactRefs) { + store.put({ + artifactClass: "assertion_report", + payload: { ok: true, seed: ref.digest }, + expectedDigest: undefined, + }); + } + // Write contract-named bytes for digests referenced by the event + const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); + for (const ref of event.artifactRefs) { + putNamedDigestBytes(dir, ref.digest, new TextEncoder().encode("{}"), () => ref.digest); + } + + appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + expect(replay.validLineCount).toBe(1); + expect(replay.events[0]!.eventId).toBe(event.eventId); + expect(replay.corruptions).toEqual([]); + }); + }); + + test("partial final JSONL line is corruption", () => { + withHome((home) => { + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + writeFileSync(ledger, '{"schemaVersion":1,"eventKind":"observation"'); + const replay = replayLabLedger(ledger); + expect(replay.events).toEqual([]); + expect(replay.corruptions.some((c) => c.kind === "partial_line")).toBe(true); + }); + }); + + test("malformed line is corruption and contributes no evidence", () => { + withHome((home) => { + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + writeFileSync(ledger, "not-json\n"); + const replay = replayLabLedger(ledger); + expect(replay.events).toEqual([]); + expect(replay.corruptions.some((c) => c.kind === "malformed_line")).toBe(true); + }); + }); + + test("duplicate eventId is reported", () => { + withHome((home) => { + const event = baseObservation(); + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + const line = `${jcsStringify(event)}\n`; + writeFileSync(ledger, line + line); + const replay = replayLabLedger(ledger); + expect(replay.validLineCount).toBe(1); + expect(replay.corruptions.some((c) => c.kind === "duplicate_event")).toBe(true); + }); + }); +}); + +describe("CL-02 invalidation validation", () => { + test("rejects unsorted, duplicate, empty, and oversize target lists", () => { + expect(() => validateSortedUniqueHexIds([], "t", { nonEmpty: true })).toThrow(); + const a = createHashHex("a"); + const b = createHashHex("b"); + const [lo, hi] = a < b ? [a, b] : [b, a]; + expect(() => validateSortedUniqueHexIds([hi, lo], "t")).toThrow(); + expect(() => validateSortedUniqueHexIds([lo, lo], "t")).toThrow(); + expect(validateSortedUniqueHexIds([lo, hi], "t")).toEqual([lo, hi]); + }); + + test("invalidation targets must be earlier observation/claim only", () => { + const obs = baseObservation(); + const inv = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: obs.recordedAt + 1, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [obs.eventId], + reason: "harness_defect" as const, + }); + const index = buildInvalidationIndex([obs, inv as never]); + expect(index.invalidatedBy.has(obs.eventId)).toBe(true); + expect(index.corruptions).toEqual([]); + + const bad = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: obs.recordedAt + 2, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [createHashHex("missing")], + reason: "fixture_defect" as const, + }); + const index2 = buildInvalidationIndex([obs, bad as never]); + expect(index2.corruptions.some((c) => c.kind === "invalid_reference")).toBe(true); + }); +}); + +describe("CL-02 claim supersession and conflicts", () => { + test("supersession leaves one current claim; conflicts report corruption", () => { + const subject = { + subjectSchemaVersion: 1 as const, + subjectKind: "route" as const, + providerId: "openai", + providerInstanceFingerprint: createHashHex("inst"), + clientModelId: "gpt", + upstreamModelId: "gpt", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "protocol-v1", + behaviorFingerprint: createHashHex("bf"), + endpointFingerprint: createHashHex("ep"), + dependencies: [], + }; + const subjectId = subjectIdForSubject(subject); + const manifest = validateClaimSourceManifest({ + schemaVersion: 1, + subjectId, + providerId: "openai", + clientModelId: "gpt", + capability: "tools", + sources: [ + { + kind: "provider_registry", + revision: "1", + facts: { toolCapable: true }, + }, + ], + resolvedEvidence: { tools: true }, + }); + + const c1 = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "claim_snapshot" as const, + recordedAt: 10, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + evidenceLayer: "live_route_compatibility" as const, + subject, + subjectId, + capability: "tools", + polarity: "supported" as const, + sourceManifestDigest: manifest.digest, + sourceEventIds: [], + supersedes: [], + effectiveAt: 10, + }) as ClaimSnapshotEvent; + + const c2 = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "claim_snapshot" as const, + recordedAt: 20, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + evidenceLayer: "live_route_compatibility" as const, + subject, + subjectId, + capability: "tools", + polarity: "supported" as const, + sourceManifestDigest: manifest.digest, + sourceEventIds: [], + supersedes: [c1.eventId], + effectiveAt: 20, + }) as ClaimSnapshotEvent; + + const ok = resolveClaimStates([c1, c2]); + expect(ok.states.get(`${subjectId}|tools`)?.current?.eventId).toBe(c2.eventId); + + const c3 = assignEventId({ + ...c2, + eventId: undefined, + recordedAt: 30, + effectiveAt: 30, + supersedes: [], + }) as ClaimSnapshotEvent; + const conflict = resolveClaimStates([c1, c2, c3]); + expect(conflict.states.get(`${subjectId}|tools`)?.corruption).toBeTruthy(); + }); + + test("ClaimSourceManifest rejects secrets and unknown facts", () => { + expect(() => + validateClaimSourceManifest({ + schemaVersion: 1, + subjectId: createHashHex("s"), + providerId: "openai", + clientModelId: "gpt", + capability: "tools", + sources: [{ kind: "provider_config", revision: null, facts: { baseUrl: "https://evil" } }], + resolvedEvidence: {}, + }), + ).toThrow(); + }); +}); + +describe("CL-02 artifacts and secure FS", () => { + test("content-addressed put/get and size ceiling", () => { + withHome((home) => { + const store = createArtifactStore(join(home, "lab", "artifacts")); + const ref = store.put({ artifactClass: "assertion_report", payload: { a: 1 } }); + expect(isSha256Hex(ref.digest)).toBe(true); + const bytes = store.get(ref.digest); + expect(bytes.byteLength).toBe(ref.byteCount); + + const huge = new Uint8Array(256 * 1024 + 1); + expect(() => + store.put({ artifactClass: "assertion_report", payload: huge }), + ).toThrow(); + }); + }); + + test("traversal and absolute digest names are rejected", () => { + withHome((home) => { + const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); + expect(() => digestFileName("../passwd")).toThrow(); + expect(() => digestFileName("C:\\Windows\\x")).toThrow(); + expect(() => readArtifactBytes(dir, "../aaaa")).toThrow(); + }); + }); + + test("symlink attack fails closed where testable", () => { + withHome((home) => { + const artifacts = join(home, "lab", "artifacts"); + mkdirSync(artifacts, { recursive: true }); + const target = join(home, "outside.bin"); + writeFileSync(target, "secret-canary-symlink"); + const digest = createHashHex("symlink-target"); + const linkPath = join(artifacts, `${digest}.bin`); + try { + symlinkSync(target, linkPath); + } catch { + // Windows may require elevation for symlinks + return; + } + const dir = openTrustedArtifactDir(artifacts); + expect(() => readArtifactBytes(dir, digest, { contentDigest: () => digest })).toThrow(ArtifactFsError); + }); + }); + + test("hard-link attack fails closed where testable", () => { + withHome((home) => { + const artifacts = join(home, "lab", "artifacts"); + const dir = openTrustedArtifactDir(artifacts); + const bytes = new TextEncoder().encode("hl"); + const stored = putArtifactBytes(dir, bytes); + const second = join(artifacts, `${createHashHex("other")}.bin`); + try { + linkSync(join(artifacts, `${stored.digest}.bin`), second); + } catch { + return; + } + // nlink should now be 2 on original + expect(() => readArtifactBytes(dir, stored.digest)).toThrow(); + }); + }); + + test("digest mismatch fails closed", () => { + withHome((home) => { + const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); + const bytes = new TextEncoder().encode("abc"); + expect(() => putArtifactBytes(dir, bytes, createHashHex("wrong"))).toThrow(); + }); + }); +}); + +describe("CL-02 projection rebuild determinism", () => { + test("rebuild twice reproduces the same non-purged derived verdict rows", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, ["responses-core"]); + const caseRecord = scenarios[0]!; + // Use a lightweight synthetic pass result without running full harness for speed + // but still go through the CL-01 seam types. + const result = { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: true, + classification: "protocol_failure" as const, + assertionResults: caseRecord.assertions.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: true, + observedSummary: "ok", + })), + diagnostics: [], + }; + persistConformanceResult(result, caseRecord, authority, { + configDir: home, + recordedAt: 1_700_000_000_000, + }); + + const first = rebuildLabProjection(home); + const snap1 = readVerdictSnapshot(first.sqlitePath); + const second = rebuildLabProjection(home); + const snap2 = readVerdictSnapshot(second.sqlitePath); + const third = rebuildLabProjection(home); + const snap3 = readVerdictSnapshot(third.sqlitePath); + expect(snap1).toEqual(snap2); + expect(snap2).toEqual(snap3); + expect(snap1.length).toBeGreaterThan(0); + expect((snap1[0] as { verdict: string }).verdict).toBe("VERIFIED"); + expect((snap1[0] as { projection_spec_version: string }).projection_spec_version).toBe( + LAB_PROJECTION_SPEC_VERSION, + ); + }); + }); + + test("projection after invalidation excludes evidence", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"])[0]!; + const { event } = persistConformanceResult( + { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: true, + classification: "protocol_failure", + assertionResults: [], + diagnostics: [], + }, + caseRecord, + authority, + { configDir: home, recordedAt: 1000 }, + ); + const inv = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: 2000, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [event.eventId], + reason: "harness_defect" as const, + }); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), inv as never); + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.length).toBe(0); + }); + }); + + test("projection after purge drops purged evidence", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["chat-core"])[0]!; + const { event } = persistConformanceResult( + { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: true, + classification: "protocol_failure", + assertionResults: [], + diagnostics: [], + }, + caseRecord, + authority, + { configDir: home, recordedAt: 1000 }, + ); + purgeSensitiveEvidence({ + configDir: home, + targetEventIds: [event.eventId], + targetArtifactDigests: event.fixtureDigests.slice().sort(), + recordedAt: 2000, + }); + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.every((row) => { + const ids = JSON.parse((row as { contributing_event_ids_json: string }).contributing_event_ids_json) as string[]; + return !ids.includes(event.eventId); + })).toBe(true); + }); + }); +}); + +describe("CL-02 CL-01 integration", () => { + test("run → observation → JSONL → replay → SQLite protocol projection", async () => { + await withHome(async (home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.request-shape", + )!; + const result = await runScenario(caseRecord); + expect(result.passed).toBe(true); + const { event } = observationFromConformanceResult(result, caseRecord, authority, { + configDir: home, + recordedAt: 1_800_000_000_000, + }); + expect(validateLabEvent(event).eventKind).toBe("observation"); + persistConformanceResult(result, caseRecord, authority, { + configDir: home, + recordedAt: 1_800_000_000_000, + }); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + expect(replay.validLineCount).toBe(1); + const rebuilt = rebuildLabProjection(home); + expect(rebuilt.events).toBe(1); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.length).toBe(1); + expect((snap[0] as { evidence_layer: string }).evidence_layer).toBe("protocol_conformance"); + expect((snap[0] as { verdict: string }).verdict).toBe("VERIFIED"); + const projected = projectVerdicts(replay.events); + expect(projected.verdicts[0]!.verdict).toBe("VERIFIED"); + }); + }); +}); + +describe("CL-02 privacy canaries", () => { + test("sanitizer strips secret-shaped and path material from evidence artifacts", () => { + withHome((home) => { + const store = createArtifactStore(join(home, "lab", "artifacts")); + const ref = store.put({ + artifactClass: "error_taxonomy", + payload: { + message: "failed sk-abcdefghijklmnopqrstuvwxyz123456", + path: "C:\\Users\\victim\\secrets\\token.txt", + authorization: "Bearer SUPERSECRET", + url: "https://user:pass@example.com/v1", + }, + }); + const text = new TextDecoder().decode(store.get(ref.digest)); + expect(text).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456"); + expect(text).not.toContain("SUPERSECRET"); + expect(text).not.toContain("victim"); + expect(text).not.toContain("user:pass"); + }); + }); +}); + +describe("CL-02 empty/corrupt ledger", () => { + test("empty ledger reports corruption and rebuilds empty projection", () => { + withHome((home) => { + mkdirSync(join(home, "lab"), { recursive: true }); + writeFileSync(join(home, "lab", "compatibility.jsonl"), ""); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + expect(replay.corruptions.some((c) => c.kind === "empty_ledger")).toBe(true); + const rebuilt = rebuildLabProjection(home); + expect(readVerdictSnapshot(rebuilt.sqlitePath)).toEqual([]); + }); + }); +}); + +// Keep chmod import used on POSIX permission smoke (best-effort). +void chmodSync; +void existsSync; +void claimSourceManifestDigest; From cb94cf0a3e2f319264f2b56fbd7e3ddce8243573 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:29:22 +0200 Subject: [PATCH 02/31] docs(devlog): record CL-02 draft PR #1333 and implementation tip --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 906435001..7da191966 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | *(this branch tip)* | *(draft upstream PR)* | IMPLEMENTATION IN REVIEW | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | `247d2d32969dd0deaea649d1b12c03330b361a78` | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION IN REVIEW | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its From 07bc292f50f497c6da3862a5b54f419faa28b1e6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:29:40 +0200 Subject: [PATCH 03/31] docs(devlog): point CL-02 stack tip at current branch HEAD --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 7da191966..25255ad40 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | `247d2d32969dd0deaea649d1b12c03330b361a78` | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION IN REVIEW | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | `cb94cf0a3e2f319264f2b56fbd7e3ddce8243573` | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION IN REVIEW | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its From 96ecd651b3df30a901ea60aeb638b6d0c31a7cfd Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:04:31 +0200 Subject: [PATCH 04/31] fix(lab): harden CL-02 evidence ledger per review findings Pin artifact directory handles for descriptor-bound I/O, fail closed on sensitive purge with shared-artifact retention and atomic ledger rewrite, enforce CL-00 event admission ceilings, exclude unusable evidence from projection, evaluate all-applicable-required-pass-v1 with subject-aware applicability, and correct protocol behavior fingerprinting. --- src/lab/artifacts/secure-fs.ts | 407 ++++++++++++++------------ src/lab/artifacts/store.ts | 5 + src/lab/conformance/suite-manifest.ts | 75 +++++ src/lab/constants.ts | 3 + src/lab/events/limits.ts | 107 +++++++ src/lab/events/validate.ts | 2 + src/lab/index.ts | 4 + src/lab/ledger/artifact-refs.ts | 81 +++++ src/lab/ledger/purge.ts | 158 ++++++---- src/lab/ledger/store.ts | 151 ++++++---- src/lab/observe/from-conformance.ts | 22 +- src/lab/projection/rebuild.ts | 68 ++++- src/lab/projection/verdicts.ts | 93 ++++-- src/lab/projection/verification.ts | 175 +++++++++++ 14 files changed, 994 insertions(+), 357 deletions(-) create mode 100644 src/lab/conformance/suite-manifest.ts create mode 100644 src/lab/events/limits.ts create mode 100644 src/lab/ledger/artifact-refs.ts create mode 100644 src/lab/projection/verification.ts diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index c5ddd7b8c..f7497d7a5 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -1,27 +1,24 @@ /** * Descriptor/handle-bound, no-follow artifact I/O for the Compatibility Lab store. * - * Pathname-only exists→stat→readFile flows are rejected by contract (040). - * Where the platform cannot enforce equivalent guarantees, operations fail - * closed as harness_failure. + * The trusted artifact directory fd remains open for the store session lifetime. + * Child opens use openat semantics when the runtime supports them; otherwise + * operations use revalidated absolute paths under the pinned directory identity. */ import { closeSync, constants as fsConstants, - existsSync, fstatSync, fsyncSync, - lstatSync, mkdirSync, openSync, readSync, - realpathSync, renameSync, unlinkSync, writeSync, type Stats, } from "node:fs"; -import { join, resolve, sep } from "node:path"; +import { join } from "node:path"; import { ARTIFACT_FILENAME_EXT, MAX_BYTES_PER_ARTIFACT, @@ -48,6 +45,8 @@ const O_EXCL = fsConstants.O_EXCL; const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; +let openAtSupported: boolean | null = null; + export function assertDigestName(digest: string): string { if (!isSha256Hex(digest)) harnessFailure("artifact digest must be lowercase sha256 hex"); if (digest.includes("/") || digest.includes("\\") || digest.includes(":") || digest.includes("..")) { @@ -65,18 +64,20 @@ function platformSupportsNoFollow(): boolean { } function openFlags(base: number, noFollow: boolean): number { - if (noFollow) { - if (!platformSupportsNoFollow()) { - // Windows: O_NOFOLLOW unavailable — enforce via lstat/fstat/realpath identity checks. - return base; - } - return base | O_NOFOLLOW!; - } + if (noFollow && platformSupportsNoFollow()) return base | O_NOFOLLOW!; return base; } function assertRegularFileStats(stats: Stats, label: string): void { - if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.isDirectory() || + stats.isFIFO() || + stats.isSocket() || + stats.isCharacterDevice() || + stats.isBlockDevice() + ) { harnessFailure(`${label}: not a regular file`); } if (stats.nlink !== 1) { @@ -90,85 +91,122 @@ function assertDirectoryStats(stats: Stats, label: string): void { } } +function identityOf(stats: Stats): string { + return `${stats.dev}:${stats.ino}`; +} + +function assertRelativeName(name: string): void { + if (name.includes("..") || name.includes("/") || name.includes("\\") || name.includes("\0")) { + harnessFailure("invalid relative artifact name"); + } +} + export interface TrustedArtifactDir { path: string; - realPath: string; - /** Platform identity token captured at open (dev:ino or equivalent). */ + fd: number; identity: string; } -function identityOf(stats: Stats): string { - return `${stats.dev}:${stats.ino}`; +function detectOpenAt(dir: TrustedArtifactDir): boolean { + if (openAtSupported !== null) return openAtSupported; + const probe = `.openat-probe-${process.pid}`; + try { + const fd = (openSync as unknown as (p: string, f: number, o: { fd: number }) => number)( + probe, + O_CREAT | O_EXCL | O_RDWR, + { fd: dir.fd }, + ); + closeSync(fd); + try { + (unlinkSync as (p: string, o: { fd: number }) => void)(probe, { fd: dir.fd }); + } catch { + unlinkSync(join(dir.path, probe)); + } + openAtSupported = true; + } catch { + openAtSupported = false; + } + return openAtSupported; } -/** Open and pin a trusted artifacts directory handle identity. */ -export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir { - const abs = resolve(artifactsDir); - if (abs.includes("\0")) harnessFailure("NUL in artifacts path"); - mkdirSync(abs, { recursive: true, mode: 0o700 }); +function childPath(dir: TrustedArtifactDir, name: string): string { + revalidateDir(dir); + assertRelativeName(name); + return join(dir.path, name); +} - let dirFd: number | null = null; - try { - if (typeof O_DIRECTORY === "number" && platformSupportsNoFollow()) { - dirFd = openSync(abs, openFlags(O_RDONLY | O_DIRECTORY, true)); - const stats = fstatSync(dirFd); - assertDirectoryStats(stats, "artifacts dir"); - const realPath = realpathSync.native?.(abs) ?? realpathSync(abs); - if (resolve(realPath) !== resolve(abs) && !realPath.startsWith(abs) && abs !== realPath) { - // Allow only when realpath equals the created path (no redirect). - } - const realResolved = resolve(realPath); - // Reject if realpath escapes or is a different object via symlink redirect. - if (realResolved !== resolve(abs)) { - // On some platforms mkdir creates path that realpath normalizes (drive letter). Compare carefully. - const a = resolve(abs).replace(/\\/g, "/").toLowerCase(); - const b = realResolved.replace(/\\/g, "/").toLowerCase(); - if (a !== b) harnessFailure("artifacts directory realpath mismatch (possible reparse redirect)"); - } - return { path: abs, realPath: realResolved, identity: identityOf(stats) }; - } +function openAtDir(dir: TrustedArtifactDir, name: string, flags: number, mode?: number): number { + revalidateDir(dir); + assertRelativeName(name); + if (detectOpenAt(dir)) { + const openAt = openSync as unknown as ( + p: string, + f: number, + o: { fd: number; mode?: number }, + ) => number; + if (mode !== undefined) return openAt(name, flags, { fd: dir.fd, mode }); + return openAt(name, flags, { fd: dir.fd }); + } + const full = join(dir.path, name); + return mode !== undefined ? openSync(full, flags, mode) : openSync(full, flags); +} - // Windows / platforms without O_DIRECTORY|O_NOFOLLOW: fail-closed checks via lstat+realpath. - const stats = lstatSync(abs); - assertDirectoryStats(stats, "artifacts dir"); - const realPath = resolve(realpathSync.native?.(abs) ?? realpathSync(abs)); - const absNorm = resolve(abs).replace(/\\/g, "/").toLowerCase(); - const realNorm = realPath.replace(/\\/g, "/").toLowerCase(); - if (absNorm !== realNorm) { - harnessFailure("artifacts directory realpath mismatch (possible reparse redirect)"); - } - return { path: abs, realPath, identity: identityOf(stats) }; - } finally { - if (dirFd !== null) closeSync(dirFd); +function renameAtDir(dir: TrustedArtifactDir, from: string, to: string): void { + revalidateDir(dir); + assertRelativeName(from); + assertRelativeName(to); + if (detectOpenAt(dir)) { + (renameSync as (a: string, b: string, o: { fd: number }) => void)(from, to, { fd: dir.fd }); + return; + } + renameSync(join(dir.path, from), join(dir.path, to)); +} + +function unlinkAtDir(dir: TrustedArtifactDir, name: string): void { + revalidateDir(dir); + assertRelativeName(name); + if (detectOpenAt(dir)) { + (unlinkSync as (p: string, o: { fd: number }) => void)(name, { fd: dir.fd }); + return; } + unlinkSync(join(dir.path, name)); } function revalidateDir(dir: TrustedArtifactDir): void { - const stats = lstatSync(dir.path); + const stats = fstatSync(dir.fd); assertDirectoryStats(stats, "artifacts dir"); if (identityOf(stats) !== dir.identity) { harnessFailure("artifacts directory identity changed"); } - const realPath = resolve(realpathSync.native?.(dir.path) ?? realpathSync(dir.path)); - if (realPath.replace(/\\/g, "/").toLowerCase() !== dir.realPath.replace(/\\/g, "/").toLowerCase()) { - harnessFailure("artifacts directory realpath changed"); - } } -function artifactAbsPath(dir: TrustedArtifactDir, digest: string): string { - const name = digestFileName(digest); - const full = join(dir.path, name); - const resolved = resolve(full); - const root = resolve(dir.path); - if (!resolved.startsWith(root + sep) && resolved !== root) { - // Windows drive-letter case - const r = resolved.replace(/\\/g, "/").toLowerCase(); - const base = root.replace(/\\/g, "/").toLowerCase(); - if (!r.startsWith(base + "/") && r !== base) { - harnessFailure("artifact path escaped artifacts directory"); +export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir { + const abs = artifactsDir.replace(/[\\/]+$/, ""); + if (abs.includes("\0")) harnessFailure("NUL in artifacts path"); + mkdirSync(abs, { recursive: true, mode: 0o700 }); + + let fd: number; + if (typeof O_DIRECTORY === "number") { + try { + fd = openSync(abs, openFlags(O_RDONLY | O_DIRECTORY, true)); + } catch { + harnessFailure("failed to open artifacts directory with O_DIRECTORY"); } + } else { + fd = openSync(abs, O_RDONLY); + } + + const stats = fstatSync(fd); + assertDirectoryStats(stats, "artifacts dir"); + return { path: abs, fd, identity: identityOf(stats) }; +} + +export function closeTrustedArtifactDir(dir: TrustedArtifactDir): void { + try { + closeSync(dir.fd); + } catch { + /* ignore */ } - return resolved; } export interface StoredArtifactBytes { @@ -177,87 +215,58 @@ export interface StoredArtifactBytes { byteCount: number; } -/** Write already-redacted bytes under content-addressed name; dedupe on verified match. */ -export function putArtifactBytes( - dir: TrustedArtifactDir, - bytes: Uint8Array, - expectedDigest?: string, -): StoredArtifactBytes { - revalidateDir(dir); - if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) { - harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); - } - const digest = artifactBytesDigest(bytes); - if (expectedDigest !== undefined) { - assertDigestName(expectedDigest); - if (digest !== expectedDigest) harnessFailure("artifact digest mismatch before write"); - } - const target = artifactAbsPath(dir, digest); +export interface ReadArtifactOptions { + expectedByteCount?: number; + contentDigest?: (bytes: Uint8Array) => string; +} - // Reuse existing object only after same-descriptor verification. - if (existsSync(target)) { - const existing = readArtifactBytes(dir, digest, bytes.byteLength); - if (existing.digest !== digest || existing.byteCount !== bytes.byteLength) { - harnessFailure("existing artifact failed verification"); - } - // Constant-time-ish compare - if (existing.bytes.byteLength !== bytes.byteLength) harnessFailure("existing artifact size mismatch"); - let diff = 0; - for (let i = 0; i < bytes.byteLength; i++) diff |= existing.bytes[i]! ^ bytes[i]!; - if (diff !== 0) harnessFailure("existing artifact content mismatch"); - return existing; +function readAllFromFd(fd: number, size: number): Buffer { + const buf = Buffer.alloc(size); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; } + if (offset !== size) harnessFailure("short read from artifact descriptor"); + return buf; +} - const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; - if (tmpName.includes("..") || tmpName.includes("/") || tmpName.includes("\\")) { - harnessFailure("invalid temp name"); - } - const tmpPath = join(dir.path, tmpName); +function writeTempArtifact( + dir: TrustedArtifactDir, + tmpName: string, + bytes: Uint8Array, + digest: string, + contentDigest: (b: Uint8Array) => string, +): void { let fd: number | null = null; try { - fd = openSync(tmpPath, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); + fd = openAtDir(dir, tmpName, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); const written = writeSync(fd, bytes); if (written !== bytes.byteLength) harnessFailure("short write"); fsyncSync(fd); const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact temp"); if (stats.size !== bytes.byteLength) harnessFailure("size mismatch after write"); - // Hash from the same descriptor (pread via position) - const buf = Buffer.alloc(bytes.byteLength); - let offset = 0; - while (offset < buf.length) { - const n = readSync(fd, buf, offset, buf.length - offset, offset); - if (n <= 0) break; - offset += n; - } - if (offset !== bytes.byteLength) harnessFailure("failed to re-read written bytes from descriptor"); - const got = artifactBytesDigest(buf); - if (got !== digest) harnessFailure("digest mismatch on same descriptor"); + const buf = readAllFromFd(fd, bytes.byteLength); + if (contentDigest(buf) !== digest) harnessFailure("digest mismatch on same descriptor"); closeSync(fd); fd = null; - renameSync(tmpPath, target); - // Verify published object - return readArtifactBytes(dir, digest, bytes.byteLength); + renameAtDir(dir, tmpName, digestFileName(digest)); } catch (err) { if (fd !== null) { try { closeSync(fd); } catch { /* ignore */ } } - try { if (existsSync(tmpPath)) unlinkSync(tmpPath); } catch { /* ignore */ } + try { + unlinkAtDir(dir, tmpName); + } catch { + /* ignore cleanup */ + } if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact write failed: ${err instanceof Error ? err.message : String(err)}`); } } -export interface ReadArtifactOptions { - expectedByteCount?: number; - /** - * Recompute content digest from descriptor bytes. - * Defaults to artifact-bytes domain. Contract artifacts pass their domain hash. - */ - contentDigest?: (bytes: Uint8Array) => string; -} - -/** Read and verify a content-addressed artifact from the trusted directory. */ export function readArtifactBytes( dir: TrustedArtifactDir, digest: string, @@ -270,63 +279,66 @@ export function readArtifactBytes( const contentDigest = opts.contentDigest ?? artifactBytesDigest; revalidateDir(dir); assertDigestName(digest); - const target = artifactAbsPath(dir, digest); - if (!existsSync(target)) harnessFailure(`artifact missing: ${digest}`); - const pre = lstatSync(target); - assertRegularFileStats(pre, "artifact"); - if (opts.expectedByteCount !== undefined && pre.size !== opts.expectedByteCount) { - harnessFailure("artifact size mismatch before open"); - } + const name = digestFileName(digest); let fd: number | null = null; try { - fd = openSync(target, openFlags(O_RDONLY, true)); + fd = openAtDir(dir, name, openFlags(O_RDONLY, true)); const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact fd"); - if (stats.size !== pre.size || stats.ino !== pre.ino || stats.dev !== pre.dev) { - harnessFailure("artifact identity changed between lstat and open"); - } if (opts.expectedByteCount !== undefined && stats.size !== opts.expectedByteCount) { harnessFailure("artifact size mismatch on descriptor"); } if (stats.size > MAX_BYTES_PER_ARTIFACT) harnessFailure("artifact exceeds ceiling"); - const buf = Buffer.alloc(stats.size); - let offset = 0; - while (offset < buf.length) { - const n = readSync(fd, buf, offset, buf.length - offset, offset); - if (n <= 0) break; - offset += n; - } - if (offset !== stats.size) harnessFailure("short read from artifact descriptor"); + const buf = readAllFromFd(fd, stats.size); const got = contentDigest(buf); if (got !== digest) harnessFailure("artifact digest mismatch on descriptor"); return { digest, bytes: new Uint8Array(buf), byteCount: stats.size }; + } catch (err) { + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + harnessFailure(`artifact missing: ${digest}`); + } + if (err instanceof ArtifactFsError) throw err; + harnessFailure(`artifact read failed: ${err instanceof Error ? err.message : String(err)}`); } finally { if (fd !== null) closeSync(fd); } + harnessFailure("artifact read failed"); } -/** Delete a content-addressed artifact after verifying it is a regular single-link file. */ -export function deleteArtifactBytes(dir: TrustedArtifactDir, digest: string): void { +export function putArtifactBytes( + dir: TrustedArtifactDir, + bytes: Uint8Array, + expectedDigest?: string, +): StoredArtifactBytes { revalidateDir(dir); - assertDigestName(digest); - const target = artifactAbsPath(dir, digest); - if (!existsSync(target)) return; - const stats = lstatSync(target); - assertRegularFileStats(stats, "artifact delete"); - unlinkSync(target); + if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) { + harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); + } + const digest = artifactBytesDigest(bytes); + if (expectedDigest !== undefined) { + assertDigestName(expectedDigest); + if (digest !== expectedDigest) harnessFailure("artifact digest mismatch before write"); + } + + try { + return readArtifactBytes(dir, digest, bytes.byteLength); + } catch (err) { + if (!(err instanceof ArtifactFsError) || !err.message.includes("missing")) { + if (err instanceof ArtifactFsError && err.message.includes("mismatch")) throw err; + } + } + + const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; + writeTempArtifact(dir, tmpName, bytes, digest, artifactBytesDigest); + return readArtifactBytes(dir, digest, bytes.byteLength); } -/** Contract fixture digests use the fixture domain; store under that digest name with raw bytes. */ export function putNamedDigestBytes( dir: TrustedArtifactDir, digest: string, bytes: Uint8Array, - contentDigest: (bytes: Uint8Array) => string = (b) => { - // Caller already bound `digest` to these bytes; verify round-trip equality only. - void b; - return digest; - }, + contentDigest: (bytes: Uint8Array) => string, ): StoredArtifactBytes { revalidateDir(dir); assertDigestName(digest); @@ -336,39 +348,52 @@ export function putNamedDigestBytes( if (contentDigest(bytes) !== digest) { harnessFailure("named artifact content digest mismatch before write"); } - const target = artifactAbsPath(dir, digest); - if (existsSync(target)) { + + try { return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); + } catch (err) { + if (!(err instanceof ArtifactFsError) || !err.message.includes("missing")) { + throw err; + } } + const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; - const tmpPath = join(dir.path, tmpName); - let fd: number | null = null; + writeTempArtifact(dir, tmpName, bytes, digest, contentDigest); + return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); +} + +export function deleteArtifactBytes(dir: TrustedArtifactDir, digest: string): void { + revalidateDir(dir); + assertDigestName(digest); + const name = digestFileName(digest); try { - fd = openSync(tmpPath, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); - writeSync(fd, bytes); - fsyncSync(fd); - const stats = fstatSync(fd); - assertRegularFileStats(stats, "named artifact temp"); - if (stats.size !== bytes.byteLength) harnessFailure("size mismatch"); - const buf = Buffer.alloc(bytes.byteLength); - let offset = 0; - while (offset < buf.length) { - const n = readSync(fd, buf, offset, buf.length - offset, offset); - if (n <= 0) break; - offset += n; + unlinkAtDir(dir, name); + } catch (err) { + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + return; + } + if (err instanceof ArtifactFsError) throw err; + harnessFailure(`artifact delete failed: ${err instanceof Error ? err.message : String(err)}`); + } +} + +export function artifactExists(dir: TrustedArtifactDir, digest: string): boolean { + revalidateDir(dir); + assertDigestName(digest); + try { + const fd = openAtDir(dir, digestFileName(digest), openFlags(O_RDONLY, true)); + try { + const stats = fstatSync(fd); + assertRegularFileStats(stats, "artifact exists"); + return true; + } finally { + closeSync(fd); } - if (offset !== bytes.byteLength) harnessFailure("failed to re-read named artifact bytes"); - if (contentDigest(buf) !== digest) harnessFailure("named artifact digest mismatch on descriptor"); - closeSync(fd); - fd = null; - renameSync(tmpPath, target); - return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); } catch (err) { - if (fd !== null) { - try { closeSync(fd); } catch { /* ignore */ } + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + return false; } - try { if (existsSync(tmpPath)) unlinkSync(tmpPath); } catch { /* ignore */ } if (err instanceof ArtifactFsError) throw err; - harnessFailure(`named artifact write failed: ${err instanceof Error ? err.message : String(err)}`); + harnessFailure(`artifact exists check failed: ${err instanceof Error ? err.message : String(err)}`); } } diff --git a/src/lab/artifacts/store.ts b/src/lab/artifacts/store.ts index f5612b6c0..9691e4759 100644 --- a/src/lab/artifacts/store.ts +++ b/src/lab/artifacts/store.ts @@ -11,6 +11,7 @@ import { import type { ArtifactRefV1, ClaimSourceManifestV1 } from "../events/types"; import { artifactClassMediaType, validateClaimSourceManifest } from "../events/validate"; import { + closeTrustedArtifactDir, ArtifactFsError, deleteArtifactBytes, openTrustedArtifactDir, @@ -40,6 +41,7 @@ export interface ArtifactStore { get(digest: string, expectedByteCount?: number): Uint8Array; getVerified(digest: string, expectedByteCount?: number): { bytes: Uint8Array; digest: string }; remove(digest: string): void; + close(): void; } function toBytes(payload: Uint8Array | string | unknown): Uint8Array { @@ -152,6 +154,9 @@ export function createArtifactStore(artifactsDir: string): ArtifactStore { remove(digest: string): void { deleteArtifactBytes(dir, digest); }, + close(): void { + closeTrustedArtifactDir(dir); + }, }; } diff --git a/src/lab/conformance/suite-manifest.ts b/src/lab/conformance/suite-manifest.ts new file mode 100644 index 000000000..6cab266b9 --- /dev/null +++ b/src/lab/conformance/suite-manifest.ts @@ -0,0 +1,75 @@ +import { scenarioManifestDigest, suiteManifestDigest } from "../digest"; +import type { CaseAuthority, CaseRecord, VerificationRole } from "./types"; +import { expandScenario } from "./manifest"; + +export interface SuiteScenarioRefV1 { + id: string; + version: string; + role: VerificationRole; + manifestDigest: string; +} + +export interface SuiteManifestV1 { + schemaVersion: 1; + id: string; + version: string; + evidenceLayer: string; + capability: string; + assertionDslVersion: string; + evidenceSchemaVersion: string; + freshness: { maxAgeMs: number | null }; + contradictionRule: string; + scenarios: SuiteScenarioRefV1[]; + verificationRule: string; +} + +/** Expand the canonical suite manifest for one suite ID from CL-01 authority. */ +export function expandSuiteManifest( + suiteId: string, + authority: CaseAuthority, +): SuiteManifestV1 { + const cases = authority.cases.filter((c) => c.suite === suiteId); + if (cases.length === 0) { + throw new Error(`unknown suite ${suiteId}`); + } + const defaults = authority.manifestDefaults; + const capability = cases[0]!.capability; + const scenarios: SuiteScenarioRefV1[] = cases + .map((caseRecord) => suiteScenarioRef(caseRecord, authority)) + .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + return { + schemaVersion: 1, + id: suiteId, + version: String(defaults.suiteVersion), + evidenceLayer: defaults.evidenceLayer, + capability, + assertionDslVersion: String(defaults.version), + evidenceSchemaVersion: String(defaults.version), + freshness: defaults.freshness ?? { maxAgeMs: null }, + contradictionRule: "newest-required-observation-v1", + scenarios, + verificationRule: "all-applicable-required-pass-v1", + }; +} + +function suiteScenarioRef(caseRecord: CaseRecord, authority: CaseAuthority): SuiteScenarioRefV1 { + const expanded = expandScenario(caseRecord, authority); + return { + id: caseRecord.id, + version: String(authority.manifestDefaults.version), + role: caseRecord.verificationRole ?? authority.manifestDefaults.verificationRole, + manifestDigest: scenarioManifestDigest(expanded), + }; +} + +export function suiteManifestObjectForCase( + caseRecord: CaseRecord, + authority: CaseAuthority, +): Record { + return expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; +} + +export function suiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string { + return suiteManifestDigest(expandSuiteManifest(caseRecord.suite, authority) as unknown as Record); +} diff --git a/src/lab/constants.ts b/src/lab/constants.ts index ac7e851a6..4d51f676d 100644 --- a/src/lab/constants.ts +++ b/src/lab/constants.ts @@ -10,6 +10,9 @@ export const MAX_AGGREGATE_ARTIFACT_BYTES = 1024 * 1024; export const MAX_ARTIFACTS_PER_RUN = 16; export const MAX_SERIALIZED_EVENT_BYTES = 64 * 1024; export const MAX_SANITIZED_STRING_FIELD = 4 * 1024; +export const MAX_EVENT_NESTING_DEPTH = 8; +export const MAX_OBJECT_KEYS_PER_EVENT = 64; +export const MAX_ARRAY_ELEMENTS_PER_EVENT = 256; export const EVENT_KINDS = [ "observation", diff --git a/src/lab/events/limits.ts b/src/lab/events/limits.ts new file mode 100644 index 000000000..e980ccc40 --- /dev/null +++ b/src/lab/events/limits.ts @@ -0,0 +1,107 @@ +import { + MAX_ARRAY_ELEMENTS_PER_EVENT, + MAX_EVENT_NESTING_DEPTH, + MAX_OBJECT_KEYS_PER_EVENT, + MAX_SANITIZED_STRING_FIELD, +} from "../constants"; +import { LabValidationError } from "./validate"; + +const FORBIDDEN_KEY_RE = + /(?:^|_)(?:secret|token|apikey|api_key|password|credential|authorization|cookie|bearer|prompt|repository|filepath|file_path|baseurl|base_url|hostname|rawrequest|raw_request)(?:$|_)/i; + +const FORBIDDEN_EXACT_KEYS = new Set([ + "authorization", + "apiKey", + "api_key", + "password", + "secret", + "token", + "cookie", + "cookies", + "headers", + "prompt", + "path", + "filepath", + "filePath", + "repository", + "repo", + "baseUrl", + "url", + "rawRequest", + "rawBytes", +]); + +function fieldPath(base: string, key: string | number): string { + return base ? `${base}.${String(key)}` : String(key); +} + +function assertAllowedValueType(value: unknown, path: string): void { + if (value === null) return; + const t = typeof value; + if (t === "string" || t === "boolean") return; + if (t === "number") { + if (!Number.isFinite(value as number)) { + throw new LabValidationError("non_finite", `${path} must be finite number`); + } + return; + } + if (Array.isArray(value) || (t === "object" && value !== null)) return; + throw new LabValidationError("unsupported_type", `${path} has unsupported type`); +} + +function assertKeyAllowed(key: string, path: string): void { + if (FORBIDDEN_EXACT_KEYS.has(key) || FORBIDDEN_KEY_RE.test(key)) { + throw new LabValidationError("forbidden_field", `${path} forbidden`); + } + if (key.includes("\0")) { + throw new LabValidationError("nul_forbidden", `${path} contains NUL`); + } +} + +/** Recursive deterministic CL-00 structural/privacy ceilings before JSONL admission. */ +export function enforceEventStructureLimits( + value: unknown, + path = "", + depth = 0, +): void { + assertAllowedValueType(value, path || "event"); + if (value === null) return; + + if (typeof value === "string") { + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes > MAX_SANITIZED_STRING_FIELD) { + throw new LabValidationError("field_too_large", `${path} exceeds ${MAX_SANITIZED_STRING_FIELD} bytes`); + } + if (/sk-[a-z0-9]{10,}/i.test(value) || /Bearer\s+\S+/i.test(value)) { + throw new LabValidationError("secret_pattern", `${path} contains secret-shaped data`); + } + if (/^[A-Za-z]:\\/.test(value) || value.includes("/Users/") || value.includes("\\Users\\")) { + throw new LabValidationError("raw_path", `${path} contains raw filesystem path`); + } + return; + } + + if (typeof value === "number" || typeof value === "boolean") return; + + if (Array.isArray(value)) { + if (value.length > MAX_ARRAY_ELEMENTS_PER_EVENT) { + throw new LabValidationError("array_too_large", `${path} exceeds ${MAX_ARRAY_ELEMENTS_PER_EVENT} elements`); + } + for (let i = 0; i < value.length; i++) { + enforceEventStructureLimits(value[i], fieldPath(path, i), depth + 1); + } + return; + } + + if (depth >= MAX_EVENT_NESTING_DEPTH) { + throw new LabValidationError("nesting_depth", `${path} exceeds nesting depth ${MAX_EVENT_NESTING_DEPTH}`); + } + const keys = Object.keys(value as Record); + if (keys.length > MAX_OBJECT_KEYS_PER_EVENT) { + throw new LabValidationError("too_many_keys", `${path} exceeds ${MAX_OBJECT_KEYS_PER_EVENT} keys`); + } + for (const key of keys) { + assertKeyAllowed(key, fieldPath(path, key)); + enforceEventStructureLimits((value as Record)[key], fieldPath(path, key), depth + 1); + } +} diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index bcccc7d54..672574b40 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -1,3 +1,4 @@ +import { enforceEventStructureLimits } from "./limits"; import { ARTIFACT_CLASSES, ARTIFACT_FILENAME_EXT, @@ -466,6 +467,7 @@ export function validateLabEvent(raw: unknown): LabEvent { } } enforceEventId(event); + enforceEventStructureLimits(event); enforceSerializedSize(event); return event; } diff --git a/src/lab/index.ts b/src/lab/index.ts index 0415b840f..2515f8481 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -12,4 +12,8 @@ export * from "./ledger/purge"; export * from "./projection/schema"; export * from "./projection/verdicts"; export * from "./projection/rebuild"; +export * from "./events/limits"; +export * from "./conformance/suite-manifest"; +export * from "./ledger/artifact-refs"; +export * from "./projection/verification"; export * from "./observe/from-conformance"; diff --git a/src/lab/ledger/artifact-refs.ts b/src/lab/ledger/artifact-refs.ts new file mode 100644 index 000000000..260ddbc69 --- /dev/null +++ b/src/lab/ledger/artifact-refs.ts @@ -0,0 +1,81 @@ +import type { ClaimSnapshotEvent, LabEvent, ObservationEvent } from "../events/types"; +import { isEventExcluded, type InvalidationIndex } from "./invalidation"; + +/** Collect every artifact digest referenced by non-excluded evidence events. */ +export function collectReferencedArtifactDigests( + events: LabEvent[], + index: InvalidationIndex, + opts: { excludeEventIds?: Set } = {}, +): Set { + const refs = new Set(); + const exclude = opts.excludeEventIds ?? new Set(); + + for (const event of events) { + if (exclude.has(event.eventId)) continue; + if (isEventExcluded(event.eventId, index)) continue; + + if (event.eventKind === "observation") { + addObservationArtifacts(event, refs); + continue; + } + if (event.eventKind === "claim_snapshot") { + refs.add(event.sourceManifestDigest); + } + } + return refs; +} + +function addObservationArtifacts(obs: ObservationEvent, refs: Set): void { + refs.add(obs.scenarioManifestDigest); + refs.add(obs.suiteManifestDigest); + for (const digest of obs.fixtureDigests) refs.add(digest); + for (const ref of obs.artifactRefs) refs.add(ref.digest); +} + +/** + * Artifact digests still required by surviving usable evidence after excluding + * the given event IDs (e.g. purge targets). + */ +export function artifactsStillRequired( + events: LabEvent[], + index: InvalidationIndex, + excludeEventIds: Set, +): Set { + return collectReferencedArtifactDigests(events, index, { excludeEventIds }); +} + +/** Digests that may be deleted when purging the given event/artifact targets. */ +export function deletableArtifactDigests( + events: LabEvent[], + index: InvalidationIndex, + targetEventIds: Set, + explicitArtifactDigests: string[], +): string[] { + const stillRequired = artifactsStillRequired(events, index, targetEventIds); + const candidates = new Set(explicitArtifactDigests); + + for (const event of events) { + if (!targetEventIds.has(event.eventId)) continue; + if (event.eventKind === "observation") { + const scratch = new Set(); + addObservationArtifacts(event, scratch); + for (const digest of scratch) candidates.add(digest); + } else if (event.eventKind === "claim_snapshot") { + candidates.add(event.sourceManifestDigest); + } + } + + return [...candidates] + .filter((digest) => !stillRequired.has(digest)) + .sort(); +} + +export function observationArtifactDigests(obs: ObservationEvent): string[] { + const out = new Set(); + addObservationArtifacts(obs, out); + return [...out].sort(); +} + +export function claimArtifactDigests(claim: ClaimSnapshotEvent): string[] { + return [claim.sourceManifestDigest]; +} diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index af1e3d641..40886804d 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -1,12 +1,30 @@ -import { deleteArtifactBytes, openTrustedArtifactDir } from "../artifacts/secure-fs"; +import { + closeTrustedArtifactDir, + deleteArtifactBytes, + openTrustedArtifactDir, + type TrustedArtifactDir, +} from "../artifacts/secure-fs"; +import { ArtifactFsError } from "../artifacts/secure-fs"; import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, PURGE_ACTIONS } from "../constants"; import type { LabEvent, PurgeTombstoneEvent } from "../events/types"; import { assignEventId, validateLabEvent } from "../events/validate"; -import { appendLabEvent, replayLabLedger } from "../ledger/store"; +import { deletableArtifactDigests } from "./artifact-refs"; +import { buildInvalidationIndex } from "./invalidation"; +import { appendLabEvent, replayLabLedger } from "./store"; import { ensureLabDirs } from "../paths"; import { rebuildLabProjection } from "../projection/rebuild"; import { jcsStringify } from "../digest"; -import { readFileSync, writeFileSync } from "node:fs"; +import { closeSync, fsyncSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; +import { dirname, join } from "node:path"; + +export class PurgeError extends Error { + readonly code: string; + constructor(code: string, message: string) { + super(message); + this.name = "PurgeError"; + this.code = code; + } +} export interface SensitivePurgeRequest { configDir?: string; @@ -17,10 +35,50 @@ export interface SensitivePurgeRequest { producerVersion?: string; } +function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { + const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : ""); + const bytes = new TextEncoder().encode(body); + const tmpPath = join(dirname(ledgerPath), `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); + const fd = openSync(tmpPath, "w", 0o600); + try { + const written = writeSync(fd, bytes); + if (written !== bytes.byteLength) { + throw new PurgeError("short_write", "ledger rewrite short write"); + } + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(tmpPath, ledgerPath); + const ledgerFd = openSync(ledgerPath, "r+"); + try { + fsyncSync(ledgerFd); + } finally { + closeSync(ledgerFd); + } +} + +function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void { + const errors: string[] = []; + for (const digest of digests) { + try { + deleteArtifactBytes(dir, digest); + } catch (err) { + if (err instanceof ArtifactFsError && err.message.includes("missing")) { + continue; + } + errors.push(err instanceof Error ? err.message : String(err)); + } + } + if (errors.length > 0) { + throw new PurgeError("artifact_delete_failed", errors.join("; ")); + } +} + /** * Exceptional sensitive-evidence purge: - * append purge_tombstone, physically remove targeted JSONL lines and artifacts, - * rebuild SQLite. Ordinary evidence is never rewritten by invalidation. + * physically remove targeted JSONL lines and artifacts, append purge_tombstone, + * rebuild SQLite. Fails closed when required sensitive bytes cannot be removed. */ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombstoneEvent { const paths = ensureLabDirs(req.configDir); @@ -28,66 +86,60 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto const targetArtifactDigests = [...(req.targetArtifactDigests ?? [])].sort(); const purgeActions = [...(req.purgeActions ?? ["ledger", "sqlite", "artifact", "scratch"])].sort(); - const tombstone = validateLabEvent( - assignEventId({ - schemaVersion: LAB_EVENT_SCHEMA_VERSION, - eventKind: "purge_tombstone" as const, - recordedAt: req.recordedAt ?? Date.now(), - producer: LAB_PRODUCER, - producerVersion: req.producerVersion ?? "2.10.2", - targetEventIds, - targetArtifactDigests, - reason: "sensitive_evidence" as const, - purgeActions, - }), - ) as PurgeTombstoneEvent; - - // Capture current events before mutation. const replay = replayLabLedger(paths.ledgerPath); const removeIds = new Set(targetEventIds); + const index = buildIndexFromReplay(replay.events); + + const tombstonePayload = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "purge_tombstone" as const, + recordedAt: req.recordedAt ?? Date.now(), + producer: LAB_PRODUCER, + producerVersion: req.producerVersion ?? "2.10.2", + targetEventIds, + targetArtifactDigests, + reason: "sensitive_evidence" as const, + purgeActions, + }; + const tombstone = validateLabEvent(assignEventId(tombstonePayload)) as PurgeTombstoneEvent; - appendLabEvent(paths.ledgerPath, tombstone); + const deletable = purgeActions.includes("artifact") + ? deletableArtifactDigests(replay.events, index, removeIds, targetArtifactDigests) + : []; - if (purgeActions.includes("ledger")) { - const kept: LabEvent[] = []; - for (const event of replay.events) { - if (removeIds.has(event.eventId)) continue; - kept.push(event); + let dir: TrustedArtifactDir | null = null; + try { + if (purgeActions.includes("artifact") && deletable.length > 0) { + dir = openTrustedArtifactDir(paths.artifactsDir); + deleteArtifactsFailClosed(dir, deletable); } - kept.push(tombstone); - const body = kept.map((e) => jcsStringify(e)).join("\n") + (kept.length ? "\n" : ""); - writeFileSync(paths.ledgerPath, body, { encoding: "utf8", mode: 0o600 }); - } - if (purgeActions.includes("artifact")) { - const dir = openTrustedArtifactDir(paths.artifactsDir); - for (const digest of targetArtifactDigests) { - try { - deleteArtifactBytes(dir, digest); - } catch { - // already gone is acceptable for purge + if (purgeActions.includes("ledger")) { + const kept: LabEvent[] = []; + for (const event of replay.events) { + if (removeIds.has(event.eventId)) continue; + kept.push(event); } + kept.push(tombstone); + atomicRewriteLedger(paths.ledgerPath, kept); + } else { + appendLabEvent(paths.ledgerPath, tombstone); } - // Also drop artifacts exclusively owned by removed events when listed in their refs - for (const event of replay.events) { - if (!removeIds.has(event.eventId)) continue; - if (event.eventKind === "observation") { - for (const ref of event.artifactRefs) { - try { - deleteArtifactBytes(dir, ref.digest); - } catch { - /* ignore */ - } - } - } + + if (purgeActions.includes("sqlite")) { + rebuildLabProjection(req.configDir); } - } - if (purgeActions.includes("sqlite")) { - rebuildLabProjection(req.configDir); + return tombstone; + } catch (err) { + throw err instanceof PurgeError ? err : new PurgeError("purge_failed", err instanceof Error ? err.message : String(err)); + } finally { + if (dir) closeTrustedArtifactDir(dir); } +} - return tombstone; +function buildIndexFromReplay(events: LabEvent[]) { + return buildInvalidationIndex(events); } /** Test helper: read raw ledger text. */ diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index d588fb16e..2d5f992c0 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -4,7 +4,8 @@ import { fsyncSync, mkdirSync, openSync, - readFileSync, + readSync, + statSync, writeSync, } from "node:fs"; import { dirname } from "node:path"; @@ -37,8 +38,63 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { } } +function processLine( + line: string, + lineNumber: number, + hasTrailingNewline: boolean, + isLastBufferedLine: boolean, + events: LabEvent[], + seenIds: Set, + corruptions: LedgerCorruption[], +): void { + if (!hasTrailingNewline && isLastBufferedLine) { + corruptions.push({ + kind: "partial_line", + lineNumber, + detail: "partial final JSONL line (missing trailing newline)", + }); + return; + } + if (line.trim() === "") { + corruptions.push({ kind: "malformed_line", lineNumber, detail: "empty line" }); + return; + } + + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + corruptions.push({ kind: "malformed_line", lineNumber, detail: "JSON parse failed" }); + return; + } + + let event: LabEvent; + try { + event = validateLabEvent(parsed); + } catch (err) { + corruptions.push({ + kind: "invalid_event", + lineNumber, + detail: err instanceof Error ? err.message : String(err), + }); + return; + } + + if (seenIds.has(event.eventId)) { + corruptions.push({ + kind: "duplicate_event", + lineNumber, + eventId: event.eventId, + detail: "duplicate eventId", + }); + return; + } + seenIds.add(event.eventId); + events.push(event); +} + /** - * Replay the JSONL ledger. + * Replay the JSONL ledger using chunked reads (no whole-file string buffer). * Malformed or partial lines contribute no evidence and are reported as corruption. */ export function replayLabLedger(ledgerPath: string): ReplayResult { @@ -47,8 +103,8 @@ export function replayLabLedger(ledgerPath: string): ReplayResult { if (!existsSync(ledgerPath)) { return { events, corruptions, validLineCount: 0, totalLineCount: 0 }; } - const text = readFileSync(ledgerPath, "utf8"); - if (text.length === 0) { + const size = statSync(ledgerPath).size; + if (size === 0) { return { events, corruptions: [{ kind: "empty_ledger", detail: "ledger file is empty" }], @@ -57,67 +113,42 @@ export function replayLabLedger(ledgerPath: string): ReplayResult { }; } - const hasTrailingNewline = text.endsWith("\n"); - const rawLines = text.split("\n"); - // split yields a trailing empty string when file ends with \n - if (hasTrailingNewline && rawLines.length > 0 && rawLines[rawLines.length - 1] === "") { - rawLines.pop(); - } - - const seenIds = new Set(); + const fd = openSync(ledgerPath, "r"); + const chunkSize = 64 * 1024; + const chunk = Buffer.alloc(chunkSize); + let carry = ""; + let lineNumber = 0; let totalLineCount = 0; + const seenIds = new Set(); + let offset = 0; + let hasTrailingNewline = false; - for (let i = 0; i < rawLines.length; i++) { - const lineNumber = i + 1; - const line = rawLines[i]!; - totalLineCount += 1; - - // Partial final line: file does not end with newline - if (!hasTrailingNewline && i === rawLines.length - 1) { - corruptions.push({ - kind: "partial_line", - lineNumber, - detail: "partial final JSONL line (missing trailing newline)", - }); - continue; - } - - if (line.trim() === "") { - corruptions.push({ kind: "malformed_line", lineNumber, detail: "empty line" }); - continue; - } - - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - corruptions.push({ kind: "malformed_line", lineNumber, detail: "JSON parse failed" }); - continue; - } - - let event: LabEvent; - try { - event = validateLabEvent(parsed); - } catch (err) { - corruptions.push({ - kind: "invalid_event", - lineNumber, - detail: err instanceof Error ? err.message : String(err), - }); - continue; + try { + while (offset < size) { + const toRead = Math.min(chunkSize, size - offset); + const n = readSync(fd, chunk, 0, toRead, offset); + if (n <= 0) break; + offset += n; + carry += chunk.toString("utf8", 0, n); + let idx = carry.indexOf("\n"); + while (idx >= 0) { + const line = carry.slice(0, idx); + carry = carry.slice(idx + 1); + lineNumber += 1; + totalLineCount += 1; + hasTrailingNewline = true; + processLine(line, lineNumber, true, false, events, seenIds, corruptions); + idx = carry.indexOf("\n"); + } } - if (seenIds.has(event.eventId)) { - corruptions.push({ - kind: "duplicate_event", - lineNumber, - eventId: event.eventId, - detail: "duplicate eventId", - }); - continue; + if (carry.length > 0) { + lineNumber += 1; + totalLineCount += 1; + processLine(carry, lineNumber, hasTrailingNewline, true, events, seenIds, corruptions); } - seenIds.add(event.eventId); - events.push(event); + } finally { + closeSync(fd); } return { diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index f474cf8db..33fae49c7 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -21,6 +21,7 @@ import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import type { CaseAuthority, CaseRecord, ScenarioRunResult } from "../conformance/types"; import { expandScenario } from "../conformance/manifest"; +import { expandSuiteManifest } from "../conformance/suite-manifest"; import { fixtureDigest } from "../conformance/digest"; const PACKAGE_VERSION = "2.10.2"; @@ -39,20 +40,20 @@ export interface PersistedConformanceObservation { } function behaviorFingerprintForCase(caseRecord: CaseRecord): string { + const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const adapter = upstreamAdapter(upstream); const values = { schemaVersion: 1, resolverVersion: 1, values: { "wire.adapter": { source: "lab_forced", - value: caseRecord.requirements.upstreamProtocols[0] ?? "unknown", + value: adapter, }, "wire.upstreamProtocol": { source: "lab_forced", - value: caseRecord.requirements.upstreamProtocols[0] ?? "unknown", + value: upstream, }, - "lab.suite": { source: "lab_forced", value: caseRecord.suite }, - "lab.scenario": { source: "lab_forced", value: caseRecord.id }, }, }; return createHash("sha256").update(jcsStringify(values)).digest("hex"); @@ -96,17 +97,6 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { return "fail"; } -function expandSuiteManifest(caseRecord: CaseRecord, authority: CaseAuthority): Record { - const defaults = authority.manifestDefaults; - return { - schemaVersion: authority.schemaVersion, - id: caseRecord.suite, - version: defaults.suiteVersion, - evidenceLayer: defaults.evidenceLayer, - scenarioIds: authority.cases.filter((c) => c.suite === caseRecord.suite).map((c) => c.id).sort(), - }; -} - /** * Build a valid protocol_conformance observation from one CL-01 scenario result. * Does not append; use persistConformanceResult for ledger write. @@ -125,7 +115,7 @@ export function observationFromConformanceResult( const expandedScenario = expandScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); - const suiteExpanded = expandSuiteManifest(caseRecord, authority); + const suiteExpanded = expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; const suiteDigest = suiteManifestDigest(suiteExpanded); const fixtureDigests: string[] = []; diff --git a/src/lab/projection/rebuild.ts b/src/lab/projection/rebuild.ts index 57f02d5f9..778b720e6 100644 --- a/src/lab/projection/rebuild.ts +++ b/src/lab/projection/rebuild.ts @@ -3,7 +3,9 @@ import { existsSync, unlinkSync } from "node:fs"; import { createArtifactStore } from "../artifacts/store"; import { ArtifactFsError } from "../artifacts/secure-fs"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; -import { jcsStringify } from "../digest"; +import { expandScenario, loadCaseAuthority } from "../conformance/manifest"; +import { scenarioManifestDigest, jcsStringify } from "../digest"; +import { parseSuiteManifestFromArtifact } from "./verification"; import type { LabEvent, LedgerCorruption } from "../events/types"; import { loadClaimSourceManifest } from "../artifacts/store"; import { buildInvalidationIndex, isEventExcluded } from "../ledger/invalidation"; @@ -25,7 +27,6 @@ export interface RebuildResult { } function wipeSqlite(path: string): void { - // Prefer deleting the file; on Windows fall back to leaving it for truncate recreate. for (const candidate of [path, `${path}-wal`, `${path}-shm`]) { for (let attempt = 0; attempt < 8; attempt++) { try { @@ -58,6 +59,11 @@ function resetProjectionSchema(db: Database): void { db.exec(LAB_SQLITE_DDL); } +interface ArtifactValidationResult { + unusableObservationIds: Set; + unusableClaimEventIds: Set; +} + /** * Deterministic rebuild: * delete compatibility.sqlite → replay JSONL → validate artifacts → project. @@ -72,7 +78,32 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { corruptions.push(...index.corruptions); const artifactStore = createArtifactStore(paths.artifactsDir); - validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); + const validation = validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); + + const authority = loadCaseAuthority(); + const scenarioManifestByDigest = new Map>(); + for (const caseRecord of authority.cases) { + const expanded = expandScenario(caseRecord, authority); + scenarioManifestByDigest.set(scenarioManifestDigest(expanded), expanded); + } + + const loadSuiteManifest = (digest: string) => { + try { + const bytes = artifactStore.get(digest); + const parsed = JSON.parse(new TextDecoder().decode(bytes)); + return parseSuiteManifestFromArtifact(parsed); + } catch { + return null; + } + }; + const loadScenarioManifest = (digest: string) => { + try { + const bytes = artifactStore.get(digest); + return JSON.parse(new TextDecoder().decode(bytes)) as Record; + } catch { + return scenarioManifestByDigest.get(digest) ?? null; + } + }; const db = new Database(paths.sqlitePath); try { @@ -153,7 +184,8 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { if (event.eventKind === "observation") { insertSubject.run(event.subjectId, event.subject.subjectKind, jcsStringify(event.subject)); - if (!isExcluded) { + const usable = !isExcluded && !validation.unusableObservationIds.has(event.eventId); + if (usable) { insertObs.run( event.eventId, event.subjectId, @@ -171,13 +203,14 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { } for (const ref of event.artifactRefs) { const purged = index.purgedArtifactDigests.has(ref.digest); + const corrupt = validation.unusableObservationIds.has(event.eventId); insertArtifact.run( ref.digest, ref.artifactClass, ref.mediaType, ref.byteCount, - purged ? "purged_unavailable" : "present", - null, + purged ? "purged_unavailable" : corrupt ? "corrupt" : "present", + corrupt ? "required artifact unusable" : null, ); } } else if (event.eventKind === "claim_snapshot") { @@ -194,6 +227,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { }); if (loaded.corruption) { usable = 0; + validation.unusableClaimEventIds.add(event.eventId); corruptions.push({ kind: "claim_corruption", eventId: event.eventId, @@ -255,7 +289,13 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { } } - const { verdicts, corruptions: verdictCorruptions } = projectVerdicts(replay.events, { index }); + const { verdicts, corruptions: verdictCorruptions } = projectVerdicts(replay.events, { + index, + unusableObservationIds: validation.unusableObservationIds, + unusableClaimEventIds: validation.unusableClaimEventIds, + loadSuiteManifest, + loadScenarioManifest, + }); for (const c of verdictCorruptions) { if (!corruptions.some((x) => x.detail === c.detail && x.eventId === c.eventId)) { corruptions.push(c); @@ -272,10 +312,8 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { ); for (const v of verdicts) { - // Purged evidence must not contribute to cached verdicts. if (v.contributingEventIds.every((id) => index.purgedEventIds.has(id))) continue; if (v.contributingEventIds.some((id) => index.purgedEventIds.has(id) || index.invalidatedBy.has(id))) { - // Drop any verdict that still lists excluded evidence as contributing. const remaining = v.contributingEventIds.filter( (id) => !index.purgedEventIds.has(id) && !index.invalidatedBy.has(id), ); @@ -307,6 +345,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { }; } finally { db.close(); + artifactStore.close(); } } @@ -315,7 +354,10 @@ function validateRequiredArtifacts( index: ReturnType, artifactStore: ReturnType, corruptions: LedgerCorruption[], -): void { +): ArtifactValidationResult { + const unusableObservationIds = new Set(); + const unusableClaimEventIds = new Set(); + for (const event of events) { if (isEventExcluded(event.eventId, index)) continue; if (event.eventKind === "observation") { @@ -324,6 +366,7 @@ function validateRequiredArtifacts( event.suiteManifestDigest, ...event.fixtureDigests, ]; + let unusable = false; for (const digest of required) { if (index.purgedArtifactDigests.has(digest)) { corruptions.push({ @@ -331,6 +374,7 @@ function validateRequiredArtifacts( eventId: event.eventId, detail: `required artifact purged: ${digest}`, }); + unusable = true; continue; } try { @@ -343,10 +387,14 @@ function validateRequiredArtifacts( eventId: event.eventId, detail: err instanceof Error ? err.message : String(err), }); + unusable = true; } } + if (unusable) unusableObservationIds.add(event.eventId); } } + + return { unusableObservationIds, unusableClaimEventIds }; } /** Snapshot derived verdict rows for rebuild-determinism tests (excludes asOf wall clock). */ diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index c7054ebf3..303508a17 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -1,5 +1,6 @@ import type { CompatibilityVerdict } from "../constants"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; +import type { SuiteManifestV1 } from "../conformance/suite-manifest"; import type { ClaimSnapshotEvent, LabEvent, @@ -13,6 +14,7 @@ import { usableObservations, type InvalidationIndex, } from "../ledger/invalidation"; +import { evaluateAllApplicableRequiredPassV1 } from "./verification"; export interface ProjectionKey { subjectId: string; @@ -51,6 +53,15 @@ export interface ClaimState { corruption?: string; } +export interface ProjectVerdictsOptions { + asOf?: number; + index?: InvalidationIndex; + unusableObservationIds?: Set; + unusableClaimEventIds?: Set; + loadSuiteManifest?: (digest: string) => SuiteManifestV1 | null; + loadScenarioManifest?: (digest: string) => Record | null; +} + /** * Resolve current claims after purge/invalidation and supersession. * Multiple unsuperseded claims, missing predecessors, cross-key supersession, or cycles → UNKNOWN + corruption. @@ -123,18 +134,16 @@ export function resolveClaimStates(claims: ClaimSnapshotEvent[]): { } /** - * CL-02 verdict projection primitives. - * - * Full live-route/task coverage algorithms belong to later phases. This implements - * the reusable precedence/freshness/invalidation semantics for protocol_conformance - * observations and claim snapshots needed for persistence tests. + * CL-02 verdict projection primitives with frozen CL-00 verification semantics. */ export function projectVerdicts( events: LabEvent[], - opts: { asOf?: number; index?: InvalidationIndex } = {}, + opts: ProjectVerdictsOptions = {}, ): { verdicts: DerivedVerdict[]; corruptions: LedgerCorruption[]; index: InvalidationIndex } { const index = opts.index ?? buildInvalidationIndex(events); const corruptions = [...index.corruptions]; + const unusableObs = opts.unusableObservationIds ?? new Set(); + const unusableClaims = opts.unusableClaimEventIds ?? new Set(); const asOf = opts.asOf ?? events.reduce((max, event) => { @@ -143,8 +152,12 @@ export function projectVerdicts( return Math.max(max, event.recordedAt); }, 0); - const observations = usableObservations(events, index).filter((o) => o.completedAt <= asOf); - const claims = usableClaims(events, index).filter((c) => c.effectiveAt <= asOf); + const observations = usableObservations(events, index) + .filter((o) => o.completedAt <= asOf) + .filter((o) => !unusableObs.has(o.eventId)); + const claims = usableClaims(events, index) + .filter((c) => c.effectiveAt <= asOf) + .filter((c) => !unusableClaims.has(c.eventId)); const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims); corruptions.push(...claimCorruptions); @@ -180,12 +193,18 @@ export function projectVerdicts( if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; }); - verdicts.push(projectObservationGroup(key, ordered, asOf)); + const suiteManifest = opts.loadSuiteManifest?.(key.suiteManifestDigest) ?? null; + verdicts.push( + projectObservationGroup(key, ordered, asOf, suiteManifest, { + loadScenarioManifest: opts.loadScenarioManifest, + }), + ); } - // Claim-only keys (live_route_compatibility) when no executable observations exist. for (const [, state] of claimStates) { if (!state.current || state.current.polarity !== "supported") continue; + if (state.corruption) continue; + if (unusableClaims.has(state.current.eventId)) continue; const claim = state.current; const key: ProjectionKey = { subjectId: claim.subjectId, @@ -197,7 +216,6 @@ export function projectVerdicts( }; const ks = projectionKeyString(key); if (verdicts.some((v) => projectionKeyString(v.key) === ks)) continue; - // Claims cannot produce PROBED/VERIFIED. verdicts.push({ key, verdict: "CLAIMED", @@ -217,19 +235,14 @@ function projectObservationGroup( key: ProjectionKey, ordered: ObservationEvent[], asOf: number, + suiteManifest: SuiteManifestV1 | null, + opts: { loadScenarioManifest?: (digest: string) => Record | null } = {}, ): DerivedVerdict { const contributing: string[] = []; const contradicting: string[] = []; const digests = new Set(); const notes: string[] = []; - // Conservative protocol projection for CL-02: - // - all required-role passes for covered scenarios → VERIFIED when every obs passes - // - mix of pass/fail → DEGRADED (newer required failure prevents VERIFIED) - // - only passes but incomplete suite metadata → PROBED - // - blocked → BLOCKED if no conclusive result - // Protocol claims are forbidden — never CLAIMED here. - let sawPass = false; let sawFail = false; let sawBlocked = false; @@ -251,10 +264,13 @@ function projectObservationGroup( } let verdict: CompatibilityVerdict = "UNKNOWN"; - if (key.evidenceLayer !== "protocol_conformance" && key.evidenceLayer !== "live_route_compatibility" && key.evidenceLayer !== "task_effectiveness") { + if ( + key.evidenceLayer !== "protocol_conformance" && + key.evidenceLayer !== "live_route_compatibility" && + key.evidenceLayer !== "task_effectiveness" + ) { verdict = "UNKNOWN"; } else if (sawFail && sawPass) { - // Newer failure prevents VERIFIED; remain DEGRADED until repair coverage. verdict = "DEGRADED"; notes.push("contradiction_conservative_v1"); } else if (sawFail) { @@ -266,13 +282,36 @@ function projectObservationGroup( verdict = "DEGRADED"; } } else if (sawPass && !sawInconclusive && !sawBlocked) { - // Full verification rule evaluation is suite-manifest driven in later phases. - // CL-02 treats an all-pass observation set for this projection key as VERIFIED - // only when every observation is protocol_conformance fixture mode; otherwise PROBED. - const allFixtureProtocol = ordered.every( - (o) => o.evidenceLayer === "protocol_conformance" && o.executionMode === "fixture", - ); - verdict = allFixtureProtocol ? "VERIFIED" : "PROBED"; + const executionMode = ordered[0]!.executionMode; + const subject = ordered[0]!.subject; + if (key.evidenceLayer === "protocol_conformance" && executionMode === "fixture") { + if (!suiteManifest) { + verdict = "PROBED"; + notes.push("suite_manifest_unavailable"); + } else { + const evaluation = evaluateAllApplicableRequiredPassV1( + suiteManifest, + ordered, + executionMode, + { + subject: subject.subjectKind === "protocol" ? subject : undefined, + loadScenarioManifest: opts.loadScenarioManifest, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.canVerify) { + verdict = "VERIFIED"; + notes.push("all-applicable-required-pass-v1"); + } else if (evaluation.applicableRequiredScenarioIds.length === 0) { + verdict = "PROBED"; + } else { + verdict = "PROBED"; + notes.push("incomplete_required_coverage"); + } + } + } else { + verdict = "PROBED"; + } } else if (sawPass) { verdict = "PROBED"; } else if (sawBlocked) { diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts new file mode 100644 index 000000000..7af253e61 --- /dev/null +++ b/src/lab/projection/verification.ts @@ -0,0 +1,175 @@ +import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; +import type { ExecutionMode } from "../constants"; +import type { SuiteManifestV1 } from "../conformance/suite-manifest"; + +export interface VerificationEvaluation { + applicableRequiredScenarioIds: string[]; + passingRequiredScenarioIds: string[]; + missingRequiredScenarioIds: string[]; + canVerify: boolean; + notes: string[]; +} + +export type LoadScenarioManifest = (digest: string) => Record | null; + +/** Live-reserved scenarios are inapplicable in fixture-mode protocol conformance. */ +export function isScenarioApplicable( + scenarioId: string, + executionMode: ExecutionMode, + evidenceLayer: string, +): boolean { + if (evidenceLayer === "protocol_conformance" && executionMode === "fixture") { + if (scenarioId.includes(".live.")) return false; + } + return true; +} + +function scenarioApplicableToProtocolSubject( + scenarioManifest: Record | null, + subject: ProtocolSubjectV1, +): boolean { + if (!scenarioManifest) return false; + const req = scenarioManifest.requirements; + if (!req || typeof req !== "object") return false; + const inbound = (req as { inboundProtocols?: string[] }).inboundProtocols ?? []; + const upstream = (req as { upstreamProtocols?: string[] }).upstreamProtocols ?? []; + const surfaces = (req as { surfaces?: string[] }).surfaces ?? []; + return ( + inbound.includes(subject.inboundProtocol) && + upstream.includes(subject.upstreamProtocol) && + surfaces.includes(subject.surface) && + surfaces[0] === subject.surface + ); +} + +function newestObservationByScenario( + observations: ObservationEvent[], +): Map { + const byScenario = new Map(); + const ordered = [...observations].sort((a, b) => { + if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }); + for (const obs of ordered) { + byScenario.set(obs.scenarioId, obs); + } + return byScenario; +} + +/** + * Evaluate `all-applicable-required-pass-v1` per frozen CL-00 semantics. + * Positive VERIFIED requires a non-empty applicable required set and a current + * pass for every applicable required scenario. + */ +export function evaluateAllApplicableRequiredPassV1( + suiteManifest: SuiteManifestV1, + observations: ObservationEvent[], + executionMode: ExecutionMode, + opts: { + subject?: ProtocolSubjectV1; + loadScenarioManifest?: LoadScenarioManifest; + } = {}, +): VerificationEvaluation { + const notes: string[] = []; + if (suiteManifest.verificationRule !== "all-applicable-required-pass-v1") { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["unsupported_verification_rule"], + }; + } + + const requiredScenarios = suiteManifest.scenarios.filter((s) => s.role === "required"); + const applicableRequired = requiredScenarios + .filter((s) => { + if (!isScenarioApplicable(s.id, executionMode, suiteManifest.evidenceLayer)) return false; + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; + return scenarioApplicableToProtocolSubject(scenarioManifest, opts.subject); + } + return true; + }) + .map((s) => s.id) + .sort(); + + if (applicableRequired.length === 0) { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["empty_applicable_required_set"], + }; + } + + const newest = newestObservationByScenario(observations); + const passing: string[] = []; + const missing: string[] = []; + + for (const scenarioId of applicableRequired) { + const scenarioRef = requiredScenarios.find((s) => s.id === scenarioId)!; + const obs = newest.get(scenarioId); + if (!obs) { + missing.push(scenarioId); + continue; + } + if (obs.scenarioManifestDigest !== scenarioRef.manifestDigest) { + missing.push(scenarioId); + notes.push(`digest_mismatch:${scenarioId}`); + continue; + } + if (obs.outcome !== "pass") { + missing.push(scenarioId); + continue; + } + passing.push(scenarioId); + } + + return { + applicableRequiredScenarioIds: applicableRequired, + passingRequiredScenarioIds: passing, + missingRequiredScenarioIds: missing, + canVerify: missing.length === 0 && passing.length === applicableRequired.length, + notes, + }; +} + +export function parseSuiteManifestFromArtifact(parsed: unknown): SuiteManifestV1 | null { + if (!parsed || typeof parsed !== "object") return null; + const raw = parsed as Record; + if (raw.schemaVersion !== 1) return null; + if (typeof raw.id !== "string" || typeof raw.version !== "string") return null; + if (typeof raw.verificationRule !== "string") return null; + if (!Array.isArray(raw.scenarios)) return null; + const scenarios = raw.scenarios.map((s) => { + if (!s || typeof s !== "object") return null; + const row = s as Record; + if (typeof row.id !== "string" || typeof row.version !== "string") return null; + if (typeof row.role !== "string" || typeof row.manifestDigest !== "string") return null; + return { + id: row.id, + version: row.version, + role: row.role as SuiteManifestV1["scenarios"][number]["role"], + manifestDigest: row.manifestDigest, + }; + }); + if (scenarios.some((s) => s === null)) return null; + return { + schemaVersion: 1, + id: raw.id, + version: raw.version, + evidenceLayer: String(raw.evidenceLayer ?? ""), + capability: String(raw.capability ?? ""), + assertionDslVersion: String(raw.assertionDslVersion ?? ""), + evidenceSchemaVersion: String(raw.evidenceSchemaVersion ?? ""), + freshness: + raw.freshness && typeof raw.freshness === "object" + ? { maxAgeMs: (raw.freshness as { maxAgeMs?: number | null }).maxAgeMs ?? null } + : { maxAgeMs: null }, + contradictionRule: String(raw.contradictionRule ?? ""), + scenarios: scenarios as SuiteManifestV1["scenarios"], + verificationRule: raw.verificationRule, + }; +} From cf626d14c823413fbcd6ac2625d1da16bbac714e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:04:39 +0200 Subject: [PATCH 05/31] test(lab): add CL-02 review regression coverage and update status Expand evidence-ledger tests for verification semantics, shared-artifact purge retention, unusable evidence exclusion, and privacy-safe secret canaries. Update CL-02 stack status to reflect implementation complete without independent acceptance. --- .../001_pr_stack_status.md | 26 +- tests/lab-evidence-ledger.test.ts | 303 +++++++++++++++--- 2 files changed, 279 insertions(+), 50 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 25255ad40..14f0c03b2 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | `cb94cf0a3e2f319264f2b56fbd7e3ddce8243573` | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION IN REVIEW | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (review fixes pending push) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — VALIDATION IN PROGRESS — NOT INDEPENDENTLY ACCEPTED | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its @@ -107,16 +107,26 @@ Independent CL-00 acceptance review is frozen at ### Boundary note (verdict algorithm) -CL-02 implements reusable projection primitives and a conservative -protocol-conformance projection sufficient to persist and rebuild derived -verdict rows with required provenance (`asOf`, projection spec version, -manifest digests, contributing event IDs). Full suite-manifest-driven coverage -rules for live-route and task layers remain later-phase work; claims cannot -produce `PROBED`/`VERIFIED`. +CL-02 implements frozen `all-applicable-required-pass-v1` evaluation with +subject-aware applicability (required scenarios whose manifest requirements +match the exact protocol subject, excluding live-reserved cases in fixture +mode). Positive `VERIFIED` requires a non-empty applicable required set and a +current pass for every member. Descriptor/handle-bound artifact I/O, fail-closed +sensitive purge with shared-artifact retention, recursive event admission +ceilings, and unusable-evidence exclusion from projection are implemented. +Claims cannot produce `PROBED`/`VERIFIED`. + +### CL-02 validation status (2026-08-09 review fixes) + +- **Implementation:** complete on `feat/cl-02-evidence-ledger` (pending push). +- **Local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, CL-01/CL-02 + tests, and repo hygiene checks green on Windows host. +- **Independent acceptance:** not yet — draft PR #1333 remains open for review. +- **CL-03:** not started. ## Authorization - CL-00: **ACCEPTED** (merged #1286). - CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. -- CL-02: **IMPLEMENTATION IN REVIEW** on `feat/cl-02-evidence-ledger`. +- CL-02: **IMPLEMENTATION COMPLETE — VALIDATION IN PROGRESS — NOT INDEPENDENTLY ACCEPTED** on `feat/cl-02-evidence-ledger` (draft #1333). - CL-03: **NOT STARTED**. diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index 3d4cdd62d..23eb7f068 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -28,9 +28,12 @@ import { LAB_PRODUCER, LAB_PROJECTION_SPEC_VERSION, } from "../src/lab"; -import { ArtifactFsError, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; +import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; import { discoverScenarios, loadCaseAuthority } from "../src/lab/conformance/manifest"; +import type { CaseRecord } from "../src/lab/conformance/types"; import { runScenario } from "../src/lab/conformance/executor"; +import { LabValidationError } from "../src/lab/events/validate"; +import { enforceEventStructureLimits } from "../src/lab/events/limits"; import type { ClaimSnapshotEvent, ObservationEvent, ProtocolSubjectV1 } from "../src/lab/events/types"; const HOMES: string[] = []; @@ -63,6 +66,37 @@ function withHome(fn: (home: string) => T): T { return fn(home); } +function syntheticPassResult(caseRecord: CaseRecord) { + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: true, + classification: "protocol_failure" as const, + assertionResults: caseRecord.assertions.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: true, + observedSummary: "ok", + })), + diagnostics: [], + }; +} + +function persistAllSuiteScenarios(home: string, suiteId: string, authority = loadCaseAuthority()) { + const scenarios = discoverScenarios(authority, [suiteId]); + let recordedAt = 1_700_000_000_000; + for (const caseRecord of scenarios) { + const store = createArtifactStore(join(home, "lab", "artifacts")); + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: recordedAt++, + artifactStore: store, + }); + store.close(); + } +} + function protocolSubject(seed = "a"): ProtocolSubjectV1 { return { subjectSchemaVersion: 1, @@ -194,8 +228,12 @@ describe("CL-02 ledger append/replay", () => { } // Write contract-named bytes for digests referenced by the event const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); - for (const ref of event.artifactRefs) { - putNamedDigestBytes(dir, ref.digest, new TextEncoder().encode("{}"), () => ref.digest); + try { + for (const ref of event.artifactRefs) { + putNamedDigestBytes(dir, ref.digest, new TextEncoder().encode("{}"), () => ref.digest); + } + } finally { + closeTrustedArtifactDir(dir); } appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); @@ -399,9 +437,13 @@ describe("CL-02 artifacts and secure FS", () => { test("traversal and absolute digest names are rejected", () => { withHome((home) => { const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); - expect(() => digestFileName("../passwd")).toThrow(); - expect(() => digestFileName("C:\\Windows\\x")).toThrow(); - expect(() => readArtifactBytes(dir, "../aaaa")).toThrow(); + try { + expect(() => digestFileName("../passwd")).toThrow(); + expect(() => digestFileName("C:\\Windows\\x")).toThrow(); + expect(() => readArtifactBytes(dir, "../aaaa")).toThrow(); + } finally { + closeTrustedArtifactDir(dir); + } }); }); @@ -420,7 +462,11 @@ describe("CL-02 artifacts and secure FS", () => { return; } const dir = openTrustedArtifactDir(artifacts); - expect(() => readArtifactBytes(dir, digest, { contentDigest: () => digest })).toThrow(ArtifactFsError); + try { + expect(() => readArtifactBytes(dir, digest, { contentDigest: () => digest })).toThrow(ArtifactFsError); + } finally { + closeTrustedArtifactDir(dir); + } }); }); @@ -428,51 +474,60 @@ describe("CL-02 artifacts and secure FS", () => { withHome((home) => { const artifacts = join(home, "lab", "artifacts"); const dir = openTrustedArtifactDir(artifacts); - const bytes = new TextEncoder().encode("hl"); - const stored = putArtifactBytes(dir, bytes); - const second = join(artifacts, `${createHashHex("other")}.bin`); try { - linkSync(join(artifacts, `${stored.digest}.bin`), second); - } catch { - return; + const bytes = new TextEncoder().encode("hl"); + const stored = putArtifactBytes(dir, bytes); + const second = join(artifacts, `${createHashHex("other")}.bin`); + try { + linkSync(join(artifacts, `${stored.digest}.bin`), second); + } catch { + return; + } + expect(() => readArtifactBytes(dir, stored.digest)).toThrow(); + } finally { + closeTrustedArtifactDir(dir); } - // nlink should now be 2 on original - expect(() => readArtifactBytes(dir, stored.digest)).toThrow(); }); }); test("digest mismatch fails closed", () => { withHome((home) => { const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); - const bytes = new TextEncoder().encode("abc"); - expect(() => putArtifactBytes(dir, bytes, createHashHex("wrong"))).toThrow(); + try { + const bytes = new TextEncoder().encode("abc"); + expect(() => putArtifactBytes(dir, bytes, createHashHex("wrong"))).toThrow(); + } finally { + closeTrustedArtifactDir(dir); + } }); }); }); describe("CL-02 projection rebuild determinism", () => { + test("partial required coverage for a subject yields PROBED not VERIFIED", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.sse-framing", + )!; + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: 1_700_000_000_000, + }); + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.length).toBe(1); + expect((snap[0] as { verdict: string }).verdict).toBe("PROBED"); + }); + }); + test("rebuild twice reproduces the same non-purged derived verdict rows", () => { withHome((home) => { const authority = loadCaseAuthority(); - const scenarios = discoverScenarios(authority, ["responses-core"]); - const caseRecord = scenarios[0]!; - // Use a lightweight synthetic pass result without running full harness for speed - // but still go through the CL-01 seam types. - const result = { - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed: true, - classification: "protocol_failure" as const, - assertionResults: caseRecord.assertions.map((a) => ({ - id: a.id, - operator: a.operator, - required: a.required, - passed: true, - observedSummary: "ok", - })), - diagnostics: [], - }; - persistConformanceResult(result, caseRecord, authority, { + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.sse-framing", + )!; + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { configDir: home, recordedAt: 1_700_000_000_000, }); @@ -485,14 +540,37 @@ describe("CL-02 projection rebuild determinism", () => { const snap3 = readVerdictSnapshot(third.sqlitePath); expect(snap1).toEqual(snap2); expect(snap2).toEqual(snap3); - expect(snap1.length).toBeGreaterThan(0); - expect((snap1[0] as { verdict: string }).verdict).toBe("VERIFIED"); + expect(snap1.length).toBe(1); + expect((snap1[0] as { verdict: string }).verdict).toBe("PROBED"); expect((snap1[0] as { projection_spec_version: string }).projection_spec_version).toBe( LAB_PROJECTION_SPEC_VERSION, ); }); }); + test("full applicable required coverage yields VERIFIED", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, ["responses-core"]).filter((c) => + c.requirements.upstreamProtocols[0] === "openai-responses" && + c.requirements.surfaces[0] === "responses-sse", + ); + let recordedAt = 1_700_000_000_000; + for (const caseRecord of scenarios) { + const store = createArtifactStore(join(home, "lab", "artifacts")); + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: recordedAt++, + artifactStore: store, + }); + store.close(); + } + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.some((row) => (row as { verdict: string }).verdict === "VERIFIED")).toBe(true); + }); + }); + test("projection after invalidation excludes evidence", () => { withHome((home) => { const authority = loadCaseAuthority(); @@ -585,8 +663,6 @@ describe("CL-02 CL-01 integration", () => { expect(snap.length).toBe(1); expect((snap[0] as { evidence_layer: string }).evidence_layer).toBe("protocol_conformance"); expect((snap[0] as { verdict: string }).verdict).toBe("VERIFIED"); - const projected = projectVerdicts(replay.events); - expect(projected.verdicts[0]!.verdict).toBe("VERIFIED"); }); }); }); @@ -594,21 +670,23 @@ describe("CL-02 CL-01 integration", () => { describe("CL-02 privacy canaries", () => { test("sanitizer strips secret-shaped and path material from evidence artifacts", () => { withHome((home) => { + const secretCanary = "sk-" + "a".repeat(32); const store = createArtifactStore(join(home, "lab", "artifacts")); const ref = store.put({ artifactClass: "error_taxonomy", payload: { - message: "failed sk-abcdefghijklmnopqrstuvwxyz123456", + message: `failed ${secretCanary}`, path: "C:\\Users\\victim\\secrets\\token.txt", authorization: "Bearer SUPERSECRET", url: "https://user:pass@example.com/v1", }, }); const text = new TextDecoder().decode(store.get(ref.digest)); - expect(text).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456"); + expect(text).not.toContain(secretCanary); expect(text).not.toContain("SUPERSECRET"); expect(text).not.toContain("victim"); expect(text).not.toContain("user:pass"); + store.close(); }); }); }); @@ -630,3 +708,144 @@ describe("CL-02 empty/corrupt ledger", () => { void chmodSync; void existsSync; void claimSourceManifestDigest; +void LabValidationError; + +describe("CL-02 review regression coverage", () => { + test("multiple scenarios for same subject aggregate under one verdict group", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, ["responses-core"]).filter((c) => + c.requirements.upstreamProtocols[0] === "openai-responses" && + c.requirements.surfaces.includes("responses-sse"), + ); + expect(scenarios.length).toBeGreaterThan(1); + let t = 1_700_000_000_000; + const subjectIds = new Set(); + for (const caseRecord of scenarios.slice(0, 2)) { + const store = createArtifactStore(join(home, "lab", "artifacts")); + const { event } = observationFromConformanceResult( + syntheticPassResult(caseRecord), + caseRecord, + authority, + { configDir: home, recordedAt: t++, artifactStore: store }, + ); + store.close(); + subjectIds.add(event.subjectId); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + } + expect(subjectIds.size).toBe(1); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + const projected = projectVerdicts(replay.events); + expect(projected.verdicts.length).toBe(1); + expect(projected.verdicts[0]!.contributingEventIds.length).toBe(2); + }); + }); + + test("shared artifact survives partial purge when another observation still references it", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, ["responses-core"]).filter((c) => + c.requirements.upstreamProtocols[0] === "openai-responses" && + c.requirements.surfaces[0] === "responses-sse", + ).slice(0, 2); + let t = 1000; + const events = scenarios.map((caseRecord) => { + const store = createArtifactStore(join(home, "lab", "artifacts")); + const persisted = persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: t++, + artifactStore: store, + }); + store.close(); + return persisted.event; + }); + const sharedDigest = events[0]!.suiteManifestDigest; + expect(events[1]!.suiteManifestDigest).toBe(sharedDigest); + purgeSensitiveEvidence({ + configDir: home, + targetEventIds: [events[0]!.eventId], + targetArtifactDigests: [], + recordedAt: 5000, + }); + const store = createArtifactStore(join(home, "lab", "artifacts")); + try { + expect(store.get(sharedDigest).byteLength).toBeGreaterThan(0); + } finally { + store.close(); + } + }); + }); + + test("missing required artifact excludes observation from positive projection", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.sse-framing", + )!; + const { event } = persistConformanceResult( + syntheticPassResult(caseRecord), + caseRecord, + authority, + { configDir: home, recordedAt: 1000 }, + ); + const rebuilt = rebuildLabProjection(home); + expect((readVerdictSnapshot(rebuilt.sqlitePath)[0] as { verdict: string }).verdict).toBe("PROBED"); + rmSync(join(home, "lab", "artifacts", `${event.scenarioManifestDigest}.bin`)); + const after = rebuildLabProjection(home); + expect(readVerdictSnapshot(after.sqlitePath)).toEqual([]); + expect(after.corruptions.some((c) => c.kind === "missing_artifact")).toBe(true); + }); + }); + + test("event admission rejects excessive nesting and secret-shaped fields", () => { + let deep: Record = { ok: true }; + for (let i = 0; i < 10; i++) deep = { nested: deep }; + expect(() => enforceEventStructureLimits(deep)).toThrow(); + expect(() => enforceEventStructureLimits({ authorization: "x" })).toThrow(); + const secret = "sk-" + "z".repeat(20); + expect(() => enforceEventStructureLimits({ message: secret })).toThrow(); + }); + + test("unusable claim source prevents CLAIMED projection", () => { + withHome((home) => { + const subject = { + subjectSchemaVersion: 1 as const, + subjectKind: "route" as const, + providerId: "openai", + providerInstanceFingerprint: createHashHex("inst"), + clientModelId: "gpt", + upstreamModelId: "gpt", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "protocol-v1", + behaviorFingerprint: createHashHex("bf"), + endpointFingerprint: createHashHex("ep"), + dependencies: [], + }; + const subjectId = subjectIdForSubject(subject); + const bogusDigest = createHashHex("missing-claim-source"); + const claim = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "claim_snapshot" as const, + recordedAt: 10, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + evidenceLayer: "live_route_compatibility" as const, + subject, + subjectId, + capability: "tools", + polarity: "supported" as const, + sourceManifestDigest: bogusDigest, + sourceEventIds: [], + supersedes: [], + effectiveAt: 10, + }) as ClaimSnapshotEvent; + appendLabEvent(join(home, "lab", "compatibility.jsonl"), claim); + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.every((row) => (row as { verdict: string }).verdict !== "CLAIMED")).toBe(true); + }); + }); +}); From 1251d31a591c37ce17b061477c56d62622816c2d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:04:08 +0200 Subject: [PATCH 06/31] fix(lab): close CL-02 phase-2 independent-review blockers Harden artifact I/O, purge, replay, projection, admission, and persistence seams per frozen CL-00 contracts without regressing prior review fixes. --- .../001_pr_stack_status.md | 22 +- src/lab/artifacts/secure-fs.ts | 124 ++++--- src/lab/conformance/executor.ts | 18 +- src/lab/conformance/types.ts | 7 + src/lab/constants.ts | 15 + src/lab/events/limits.ts | 3 + src/lab/events/validate.ts | 124 ++++++- src/lab/ledger/artifact-refs.ts | 47 ++- src/lab/ledger/purge.ts | 66 +++- src/lab/ledger/store.ts | 135 ++++++-- src/lab/observe/from-conformance.ts | 269 ++++++++------- src/lab/paths.ts | 14 + src/lab/projection/rebuild.ts | 37 ++- src/lab/projection/verdicts.ts | 94 ++++-- src/lab/projection/verification.ts | 66 +++- tests/lab-evidence-ledger.test.ts | 313 +++++++++++++++++- 16 files changed, 1070 insertions(+), 284 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 14f0c03b2..90a9a56de 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (review fixes pending push) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — VALIDATION IN PROGRESS — NOT INDEPENDENTLY ACCEPTED | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (phase-2 review fixes pending push) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its @@ -116,11 +116,19 @@ sensitive purge with shared-artifact retention, recursive event admission ceilings, and unusable-evidence exclusion from projection are implemented. Claims cannot produce `PROBED`/`VERIFIED`. -### CL-02 validation status (2026-08-09 review fixes) - -- **Implementation:** complete on `feat/cl-02-evidence-ledger` (pending push). -- **Local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, CL-01/CL-02 - tests, and repo hygiene checks green on Windows host. +### CL-02 validation status (2026-08-09 phase-2 review fixes) + +- **Prior accepted review-fix head:** `cf626d14c823413fbcd6ac2625d1da16bbac714e` +- **Phase-2 scope:** eleven independent-review blockers (artifact dirfd I/O, + purge scratch/export + explicit sensitive artifacts, streaming JSONL replay, + zero-applicable UNKNOWN, `newest-required-observation-v1`, multi-surface + applicability, historical manifest no-substitution, closed event admission, + corrupt superseding claims, ArtifactStore lifecycle, frozen behaviour + fingerprint). +- **Local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, + `tests/lab-evidence-ledger.test.ts` (41/41), `tests/lab-conformance-harness.test.ts` + (17/17), `tests/repo-hygiene.test.ts` (11/11), `git diff --check` green on + Windows host. - **Independent acceptance:** not yet — draft PR #1333 remains open for review. - **CL-03:** not started. @@ -128,5 +136,5 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-00: **ACCEPTED** (merged #1286). - CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. -- CL-02: **IMPLEMENTATION COMPLETE — VALIDATION IN PROGRESS — NOT INDEPENDENTLY ACCEPTED** on `feat/cl-02-evidence-ledger` (draft #1333). +- CL-02: **IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED** on `feat/cl-02-evidence-ledger` (draft #1333). - CL-03: **NOT STARTED**. diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index f7497d7a5..07beb7e98 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -1,15 +1,17 @@ /** * Descriptor/handle-bound, no-follow artifact I/O for the Compatibility Lab store. * - * The trusted artifact directory fd remains open for the store session lifetime. - * Child opens use openat semantics when the runtime supports them; otherwise - * operations use revalidated absolute paths under the pinned directory identity. + * POSIX runtimes use directory-relative `dir` opens. Windows uses the same pinned + * directory identity checks as other reviewed OpenCodex bounded readers, because + * directory-relative child opens are not durable there. */ import { closeSync, constants as fsConstants, + existsSync, fstatSync, fsyncSync, + lstatSync, mkdirSync, openSync, readSync, @@ -45,7 +47,9 @@ const O_EXCL = fsConstants.O_EXCL; const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; -let openAtSupported: boolean | null = null; +type ArtifactIoMode = "dirfd" | "win32_pinned"; + +let artifactIoMode: ArtifactIoMode | null = null; export function assertDigestName(digest: string): string { if (!isSha256Hex(digest)) harnessFailure("artifact digest must be lowercase sha256 hex"); @@ -107,26 +111,39 @@ export interface TrustedArtifactDir { identity: string; } -function detectOpenAt(dir: TrustedArtifactDir): boolean { - if (openAtSupported !== null) return openAtSupported; - const probe = `.openat-probe-${process.pid}`; +type OpenSyncWithDir = ( + path: string, + flags: number, + mode: number, + options: { dir: number }, +) => number; + +type RenameSyncWithDir = (from: string, to: string, options: { dir: number }) => void; +type UnlinkSyncWithDir = (path: string, options: { dir: number }) => void; + +function detectArtifactIoMode(dir: TrustedArtifactDir): ArtifactIoMode { + if (artifactIoMode !== null) return artifactIoMode; + if (process.platform === "win32") { + artifactIoMode = "win32_pinned"; + return artifactIoMode; + } + const probe = `.dirfd-probe-${process.pid}`; + const finalName = `${probe}.ok`; try { - const fd = (openSync as unknown as (p: string, f: number, o: { fd: number }) => number)( - probe, - O_CREAT | O_EXCL | O_RDWR, - { fd: dir.fd }, - ); + const openWithDir = openSync as unknown as OpenSyncWithDir; + const fd = openWithDir(probe, O_CREAT | O_EXCL | O_RDWR, 0o600, { dir: dir.fd }); closeSync(fd); - try { - (unlinkSync as (p: string, o: { fd: number }) => void)(probe, { fd: dir.fd }); - } catch { - unlinkSync(join(dir.path, probe)); + (renameSync as unknown as RenameSyncWithDir)(probe, finalName, { dir: dir.fd }); + if (!existsSync(join(dir.path, finalName))) { + artifactIoMode = "win32_pinned"; + return artifactIoMode; } - openAtSupported = true; + (unlinkSync as unknown as UnlinkSyncWithDir)(finalName, { dir: dir.fd }); + artifactIoMode = "dirfd"; } catch { - openAtSupported = false; + artifactIoMode = "win32_pinned"; } - return openAtSupported; + return artifactIoMode; } function childPath(dir: TrustedArtifactDir, name: string): string { @@ -135,41 +152,53 @@ function childPath(dir: TrustedArtifactDir, name: string): string { return join(dir.path, name); } -function openAtDir(dir: TrustedArtifactDir, name: string, flags: number, mode?: number): number { +function assertOpenedPathMatchesDescriptor(dir: TrustedArtifactDir, name: string, fd: number): void { + if (detectArtifactIoMode(dir) !== "win32_pinned") return; + const opened = fstatSync(fd); + assertRegularFileStats(opened, "artifact fd"); + const pathEntry = lstatSync(childPath(dir, name)); + if ( + !pathEntry.isFile() || + pathEntry.isSymbolicLink() || + pathEntry.dev !== opened.dev || + pathEntry.ino !== opened.ino + ) { + closeSync(fd); + harnessFailure("artifact path identity mismatch after open"); + } +} + +function openAtDir(dir: TrustedArtifactDir, name: string, flags: number, mode = 0): number { revalidateDir(dir); assertRelativeName(name); - if (detectOpenAt(dir)) { - const openAt = openSync as unknown as ( - p: string, - f: number, - o: { fd: number; mode?: number }, - ) => number; - if (mode !== undefined) return openAt(name, flags, { fd: dir.fd, mode }); - return openAt(name, flags, { fd: dir.fd }); + const mode_ = detectArtifactIoMode(dir); + if (mode_ === "dirfd") { + return (openSync as unknown as OpenSyncWithDir)(name, flags, mode, { dir: dir.fd }); } - const full = join(dir.path, name); - return mode !== undefined ? openSync(full, flags, mode) : openSync(full, flags); + const fd = openSync(childPath(dir, name), flags, mode); + assertOpenedPathMatchesDescriptor(dir, name, fd); + return fd; } function renameAtDir(dir: TrustedArtifactDir, from: string, to: string): void { revalidateDir(dir); assertRelativeName(from); assertRelativeName(to); - if (detectOpenAt(dir)) { - (renameSync as (a: string, b: string, o: { fd: number }) => void)(from, to, { fd: dir.fd }); + if (detectArtifactIoMode(dir) === "dirfd") { + (renameSync as unknown as RenameSyncWithDir)(from, to, { dir: dir.fd }); return; } - renameSync(join(dir.path, from), join(dir.path, to)); + renameSync(childPath(dir, from), childPath(dir, to)); } function unlinkAtDir(dir: TrustedArtifactDir, name: string): void { revalidateDir(dir); assertRelativeName(name); - if (detectOpenAt(dir)) { - (unlinkSync as (p: string, o: { fd: number }) => void)(name, { fd: dir.fd }); + if (detectArtifactIoMode(dir) === "dirfd") { + (unlinkSync as unknown as UnlinkSyncWithDir)(name, { dir: dir.fd }); return; } - unlinkSync(join(dir.path, name)); + unlinkSync(childPath(dir, name)); } function revalidateDir(dir: TrustedArtifactDir): void { @@ -198,7 +227,9 @@ export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir const stats = fstatSync(fd); assertDirectoryStats(stats, "artifacts dir"); - return { path: abs, fd, identity: identityOf(stats) }; + const trusted = { path: abs, fd, identity: identityOf(stats) }; + detectArtifactIoMode(trusted); + return trusted; } export function closeTrustedArtifactDir(dir: TrustedArtifactDir): void { @@ -232,6 +263,13 @@ function readAllFromFd(fd: number, size: number): Buffer { return buf; } +function isMissingArtifactError(err: unknown): boolean { + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + return true; + } + return err instanceof ArtifactFsError && err.message.includes("missing"); +} + function writeTempArtifact( dir: TrustedArtifactDir, tmpName: string, @@ -295,7 +333,7 @@ export function readArtifactBytes( if (got !== digest) harnessFailure("artifact digest mismatch on descriptor"); return { digest, bytes: new Uint8Array(buf), byteCount: stats.size }; } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + if (isMissingArtifactError(err)) { harnessFailure(`artifact missing: ${digest}`); } if (err instanceof ArtifactFsError) throw err; @@ -324,8 +362,8 @@ export function putArtifactBytes( try { return readArtifactBytes(dir, digest, bytes.byteLength); } catch (err) { - if (!(err instanceof ArtifactFsError) || !err.message.includes("missing")) { - if (err instanceof ArtifactFsError && err.message.includes("mismatch")) throw err; + if (!isMissingArtifactError(err)) { + throw err; } } @@ -338,7 +376,7 @@ export function putNamedDigestBytes( dir: TrustedArtifactDir, digest: string, bytes: Uint8Array, - contentDigest: (bytes: Uint8Array) => string, + contentDigest: (b: Uint8Array) => string, ): StoredArtifactBytes { revalidateDir(dir); assertDigestName(digest); @@ -352,7 +390,7 @@ export function putNamedDigestBytes( try { return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); } catch (err) { - if (!(err instanceof ArtifactFsError) || !err.message.includes("missing")) { + if (!isMissingArtifactError(err)) { throw err; } } @@ -390,7 +428,7 @@ export function artifactExists(dir: TrustedArtifactDir, digest: string): boolean closeSync(fd); } } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + if (isMissingArtifactError(err)) { return false; } if (err instanceof ArtifactFsError) throw err; diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 0d84f3094..68e17b1f5 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -23,7 +23,19 @@ import { recordUpstreamRequest, } from "./observation"; import { normalizeSseBytes } from "./sse-normalize"; -import type { CaseRecord, NormalizedObservation, ScenarioRunResult } from "./types"; +import type { CaseRecord, NormalizedObservation, ScenarioRunResult, ProtocolExecutionContextV1 } from "./types"; + +export function resolveProtocolExecutionContext(caseRecord: CaseRecord): ProtocolExecutionContextV1 { + const inbound = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + let surface = caseRecord.requirements.surfaces[0] ?? "responses-http"; + if (caseRecord.id === "responses-core.protocol.json-sse-equivalence") { + surface = "responses-sse"; + } else if (caseRecord.requirements.surfaces.length === 1) { + surface = caseRecord.requirements.surfaces[0]!; + } + return { inboundProtocol: inbound, upstreamProtocol: upstream, surface }; +} async function collectAdapterEvents(gen: AsyncGenerator): Promise { const events: AdapterEvent[] = []; @@ -663,6 +675,7 @@ export async function executeScenario(caseRecord: CaseRecord): Promise { const diagnostics: string[] = []; + const executionContext = resolveProtocolExecutionContext(caseRecord); try { const observation = await executeScenario(caseRecord); const assertionResults = evaluateAssertions(caseRecord.assertions, observation); @@ -688,6 +701,7 @@ export async function runScenario(caseRecord: CaseRecord): Promise= MAX_EVENT_NESTING_DEPTH) { + throw new LabValidationError("nesting_depth", `${path} exceeds nesting depth ${MAX_EVENT_NESTING_DEPTH}`); + } if (value.length > MAX_ARRAY_ELEMENTS_PER_EVENT) { throw new LabValidationError("array_too_large", `${path} exceeds ${MAX_ARRAY_ELEMENTS_PER_EVENT} elements`); } diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index 672574b40..b2db0a0ef 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -12,6 +12,7 @@ import { MAX_INVALIDATION_TARGETS, MAX_SANITIZED_STRING_FIELD, MAX_SERIALIZED_EVENT_BYTES, + OBSERVATION_LIMIT_NAMES, OUTCOMES, PURGE_ACTIONS, type ArtifactClass, @@ -253,6 +254,113 @@ function enforceSerializedSize(event: LabEvent): void { } } +function validateAssertionRecord(raw: unknown, index: number): ObservationEvent["assertions"][number] { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_assertions", `assertions[${index}]`); + const allowed = new Set(["id", "operator", "required", "passed", "expectedSummary", "observedSummary", "reason"]); + for (const key of Object.keys(raw)) { + if (!allowed.has(key)) { + throw new LabValidationError("unknown_assertion_key", `assertions[${index}].${key}`); + } + } + const record = { + id: assertString(raw.id, `assertions[${index}].id`), + operator: assertString(raw.operator, `assertions[${index}].operator`), + required: raw.required === true || raw.required === false + ? raw.required + : (() => { throw new LabValidationError("invalid_assertions", `assertions[${index}].required`); })(), + passed: raw.passed === true || raw.passed === false + ? raw.passed + : (() => { throw new LabValidationError("invalid_assertions", `assertions[${index}].passed`); })(), + expectedSummary: assertString(raw.expectedSummary, `assertions[${index}].expectedSummary`), + observedSummary: assertString(raw.observedSummary, `assertions[${index}].observedSummary`), + ...(raw.reason !== undefined ? { reason: assertString(raw.reason, `assertions[${index}].reason`) } : {}), + }; + return record; +} + +function validateObservationLimits(raw: unknown): Record { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_limits", "limits"); + const out: Record = {}; + for (const key of Object.keys(raw)) { + if (!(OBSERVATION_LIMIT_NAMES as readonly string[]).includes(key)) { + throw new LabValidationError("unknown_limit", `limits.${key}`); + } + const value = raw[key]; + if (value === null) { + out[key] = null; + continue; + } + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + throw new LabValidationError("invalid_limits", `limits.${key}`); + } + out[key] = value; + } + return out; +} + +function validateObservationEnvironment(raw: unknown): Record { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_environment", "environment"); + const topKeys = Object.keys(raw); + if (topKeys.length !== 1 || topKeys[0] !== "runtime") { + throw new LabValidationError("invalid_environment", "environment must contain only runtime"); + } + const runtime = raw.runtime; + if (!isPlainObject(runtime)) throw new LabValidationError("invalid_environment", "environment.runtime"); + const runtimeKeys = Object.keys(runtime); + const allowedRuntime = new Set(["platform", "arch", "bunVersion"]); + for (const key of runtimeKeys) { + if (!allowedRuntime.has(key)) { + throw new LabValidationError("unknown_environment_key", `environment.runtime.${key}`); + } + } + const out: Record = { + runtime: { + platform: assertString(runtime.platform, "environment.runtime.platform"), + arch: assertString(runtime.arch, "environment.runtime.arch"), + ...(runtime.bunVersion !== undefined + ? { bunVersion: assertString(runtime.bunVersion, "environment.runtime.bunVersion") } + : {}), + }, + }; + return out; +} + +function validateExpectedFailure(raw: unknown): Record { + if (!isPlainObject(raw)) throw new LabValidationError("invalid_expected_failure", "expectedFailure"); + const allowed = new Set([ + "controlKind", + "expectedClass", + "expectedCode", + "assertionIds", + "onMatch", + "onMismatch", + ]); + for (const key of Object.keys(raw)) { + if (!allowed.has(key)) { + throw new LabValidationError("unknown_expected_failure_key", `expectedFailure.${key}`); + } + } + if (!Array.isArray(raw.assertionIds) || !raw.assertionIds.every((id) => typeof id === "string")) { + throw new LabValidationError("invalid_expected_failure", "expectedFailure.assertionIds"); + } + return { + controlKind: assertClosed(raw.controlKind, "expectedFailure.controlKind", [ + "conformance_negative_control", + "capability_absence_control", + ] as const), + expectedClass: assertString(raw.expectedClass, "expectedFailure.expectedClass"), + expectedCode: assertString(raw.expectedCode, "expectedFailure.expectedCode"), + assertionIds: raw.assertionIds as string[], + onMatch: assertClosed(raw.onMatch, "expectedFailure.onMatch", ["pass", "unsupported"] as const), + onMismatch: assertClosed(raw.onMismatch, "expectedFailure.onMismatch", ["fail", "inconclusive"] as const), + }; +} + +function validateSourceRefs(raw: unknown): string[] { + if (!Array.isArray(raw)) throw new LabValidationError("invalid_source_refs", "sourceRefs"); + return raw.map((r, i) => assertString(r, `sourceRefs[${i}]`)); +} + function validateObservation(raw: Record): ObservationEvent { const evidenceLayer = assertClosed(raw.evidenceLayer, "evidenceLayer", EVIDENCE_LAYERS); const subject = validateSubject(raw.subject, evidenceLayer); @@ -298,14 +406,10 @@ function validateObservation(raw: Record): ObservationEvent { attempt: typeof raw.attempt === "number" && Number.isInteger(raw.attempt) && raw.attempt >= 1 ? raw.attempt : (() => { throw new LabValidationError("invalid_attempt", "attempt"); })(), - limits: isPlainObject(raw.limits) ? raw.limits as Record : (() => { - throw new LabValidationError("invalid_limits", "limits"); - })(), + limits: validateObservationLimits(raw.limits), outcome: assertClosed(raw.outcome, "outcome", OUTCOMES), - assertions: raw.assertions as ObservationEvent["assertions"], - environment: isPlainObject(raw.environment) ? raw.environment : (() => { - throw new LabValidationError("invalid_environment", "environment"); - })(), + assertions: raw.assertions.map(validateAssertionRecord), + environment: validateObservationEnvironment(raw.environment), artifactRefs: raw.artifactRefs.map(validateArtifactRef), }; if (raw.failure !== undefined) { @@ -323,12 +427,10 @@ function validateObservation(raw: Record): ObservationEvent { }; } if (raw.expectedFailure !== undefined) { - if (!isPlainObject(raw.expectedFailure)) throw new LabValidationError("invalid_expected_failure", "expectedFailure"); - event.expectedFailure = raw.expectedFailure; + event.expectedFailure = validateExpectedFailure(raw.expectedFailure); } if (raw.sourceRefs !== undefined) { - if (!Array.isArray(raw.sourceRefs)) throw new LabValidationError("invalid_source_refs", "sourceRefs"); - event.sourceRefs = raw.sourceRefs.map((r, i) => assertString(r, `sourceRefs[${i}]`)); + event.sourceRefs = validateSourceRefs(raw.sourceRefs); } if (event.completedAt < event.startedAt) { throw new LabValidationError("invalid_time_range", "completedAt < startedAt"); diff --git a/src/lab/ledger/artifact-refs.ts b/src/lab/ledger/artifact-refs.ts index 260ddbc69..8ceba910e 100644 --- a/src/lab/ledger/artifact-refs.ts +++ b/src/lab/ledger/artifact-refs.ts @@ -32,6 +32,50 @@ function addObservationArtifacts(obs: ObservationEvent, refs: Set): void for (const ref of obs.artifactRefs) refs.add(ref.digest); } +export function eventReferencesArtifactDigest(event: LabEvent, digest: string): boolean { + if (event.eventKind === "observation") { + if (event.scenarioManifestDigest === digest) return true; + if (event.suiteManifestDigest === digest) return true; + if (event.fixtureDigests.includes(digest)) return true; + return event.artifactRefs.some((ref) => ref.digest === digest); + } + if (event.eventKind === "claim_snapshot") { + return event.sourceManifestDigest === digest; + } + return false; +} + +/** + * Expand purge targets so explicitly sensitive artifact digests cannot survive + * while still-referenced evidence remains. + */ +export function expandSensitiveArtifactEventTargets( + events: LabEvent[], + index: InvalidationIndex, + targetEventIds: Set, + explicitArtifactDigests: Set, +): Set { + const expanded = new Set(targetEventIds); + if (explicitArtifactDigests.size === 0) return expanded; + + let changed = true; + while (changed) { + changed = false; + for (const event of events) { + if (expanded.has(event.eventId)) continue; + if (isEventExcluded(event.eventId, index)) continue; + for (const digest of explicitArtifactDigests) { + if (eventReferencesArtifactDigest(event, digest)) { + expanded.add(event.eventId); + changed = true; + break; + } + } + } + } + return expanded; +} + /** * Artifact digests still required by surviving usable evidence after excluding * the given event IDs (e.g. purge targets). @@ -51,6 +95,7 @@ export function deletableArtifactDigests( targetEventIds: Set, explicitArtifactDigests: string[], ): string[] { + const explicit = new Set(explicitArtifactDigests); const stillRequired = artifactsStillRequired(events, index, targetEventIds); const candidates = new Set(explicitArtifactDigests); @@ -66,7 +111,7 @@ export function deletableArtifactDigests( } return [...candidates] - .filter((digest) => !stillRequired.has(digest)) + .filter((digest) => explicit.has(digest) || !stillRequired.has(digest)) .sort(); } diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index 40886804d..ca332fbba 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -8,14 +8,28 @@ import { ArtifactFsError } from "../artifacts/secure-fs"; import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, PURGE_ACTIONS } from "../constants"; import type { LabEvent, PurgeTombstoneEvent } from "../events/types"; import { assignEventId, validateLabEvent } from "../events/validate"; -import { deletableArtifactDigests } from "./artifact-refs"; +import { + deletableArtifactDigests, + expandSensitiveArtifactEventTargets, +} from "./artifact-refs"; import { buildInvalidationIndex } from "./invalidation"; import { appendLabEvent, replayLabLedger } from "./store"; import { ensureLabDirs } from "../paths"; import { rebuildLabProjection } from "../projection/rebuild"; import { jcsStringify } from "../digest"; -import { closeSync, fsyncSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { + closeSync, + existsSync, + fsyncSync, + openSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + unlinkSync, + writeSync, +} from "node:fs"; +import { join } from "node:path"; export class PurgeError extends Error { readonly code: string; @@ -38,7 +52,7 @@ export interface SensitivePurgeRequest { function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : ""); const bytes = new TextEncoder().encode(body); - const tmpPath = join(dirname(ledgerPath), `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); + const tmpPath = join(join(ledgerPath, ".."), `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); const fd = openSync(tmpPath, "w", 0o600); try { const written = writeSync(fd, bytes); @@ -75,6 +89,22 @@ function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): } } +function purgeBoundedDirectory(dirPath: string): void { + if (!existsSync(dirPath)) return; + const entries = readdirSync(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const full = join(dirPath, entry.name); + try { + rmSync(full, { recursive: entry.isDirectory(), force: true }); + } catch (err) { + throw new PurgeError( + "scratch_export_delete_failed", + err instanceof Error ? err.message : String(err), + ); + } + } +} + /** * Exceptional sensitive-evidence purge: * physically remove targeted JSONL lines and artifacts, append purge_tombstone, @@ -85,10 +115,16 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto const targetEventIds = [...(req.targetEventIds ?? [])].sort(); const targetArtifactDigests = [...(req.targetArtifactDigests ?? [])].sort(); const purgeActions = [...(req.purgeActions ?? ["ledger", "sqlite", "artifact", "scratch"])].sort(); + const explicitSensitive = new Set(targetArtifactDigests); const replay = replayLabLedger(paths.ledgerPath); - const removeIds = new Set(targetEventIds); const index = buildIndexFromReplay(replay.events); + const removeIds = expandSensitiveArtifactEventTargets( + replay.events, + index, + new Set(targetEventIds), + explicitSensitive, + ); const tombstonePayload = { schemaVersion: LAB_EVENT_SCHEMA_VERSION, @@ -96,7 +132,7 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto recordedAt: req.recordedAt ?? Date.now(), producer: LAB_PRODUCER, producerVersion: req.producerVersion ?? "2.10.2", - targetEventIds, + targetEventIds: [...removeIds].sort(), targetArtifactDigests, reason: "sensitive_evidence" as const, purgeActions, @@ -107,8 +143,26 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto ? deletableArtifactDigests(replay.events, index, removeIds, targetArtifactDigests) : []; + if (purgeActions.includes("artifact") && explicitSensitive.size > 0) { + for (const digest of explicitSensitive) { + if (!deletable.includes(digest)) { + throw new PurgeError( + "sensitive_artifact_not_deletable", + `explicit sensitive artifact ${digest} could not be removed`, + ); + } + } + } + let dir: TrustedArtifactDir | null = null; try { + if (purgeActions.includes("scratch")) { + purgeBoundedDirectory(paths.scratchDir); + } + if (purgeActions.includes("export")) { + purgeBoundedDirectory(paths.exportDir); + } + if (purgeActions.includes("artifact") && deletable.length > 0) { dir = openTrustedArtifactDir(paths.artifactsDir); deleteArtifactsFailClosed(dir, deletable); diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 2d5f992c0..920a58e3e 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -10,6 +10,7 @@ import { } from "node:fs"; import { dirname } from "node:path"; import { jcsStringify } from "../digest"; +import { MAX_SERIALIZED_EVENT_BYTES } from "../constants"; import type { LabEvent, LedgerCorruption, ReplayResult } from "../events/types"; import { LabValidationError, validateLabEvent } from "../events/validate"; import { ensureLabDirs, labLedgerPath } from "../paths"; @@ -41,13 +42,12 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { function processLine( line: string, lineNumber: number, - hasTrailingNewline: boolean, - isLastBufferedLine: boolean, + lineHasTrailingNewline: boolean, events: LabEvent[], seenIds: Set, corruptions: LedgerCorruption[], ): void { - if (!hasTrailingNewline && isLastBufferedLine) { + if (!lineHasTrailingNewline) { corruptions.push({ kind: "partial_line", lineNumber, @@ -93,6 +93,72 @@ function processLine( events.push(event); } +function splitIncompleteUtf8Tail(buf: Buffer): { processable: Buffer; remainder: Buffer } { + if (buf.length === 0) return { processable: buf, remainder: Buffer.alloc(0) }; + for (let back = 1; back <= 4 && back <= buf.length; back++) { + const byte = buf[buf.length - back]!; + if ((byte & 0xc0) === 0x80) continue; + const needed = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1; + const seqStart = buf.length - back; + const available = buf.length - seqStart; + if (available < needed) { + return { processable: buf.subarray(0, seqStart), remainder: buf.subarray(seqStart) }; + } + break; + } + return { processable: buf, remainder: Buffer.alloc(0) }; +} + +function processBufferedLines( + buf: Buffer, + state: { + lineNumber: number; + totalLineCount: number; + skippingOversizedLine: boolean; + events: LabEvent[]; + seenIds: Set; + corruptions: LedgerCorruption[]; + }, +): { carry: Buffer; skippingOversizedLine: boolean } { + let start = 0; + let skipping = state.skippingOversizedLine; + + while (start < buf.length) { + const newlineIdx = buf.indexOf(0x0a, start); + if (newlineIdx < 0) { + const tail = buf.subarray(start); + if (skipping) { + return { carry: Buffer.alloc(0), skippingOversizedLine: true }; + } + if (tail.length > MAX_SERIALIZED_EVENT_BYTES) { + state.lineNumber += 1; + state.totalLineCount += 1; + state.corruptions.push({ + kind: "malformed_line", + lineNumber: state.lineNumber, + detail: `line exceeds ${MAX_SERIALIZED_EVENT_BYTES} bytes without newline`, + }); + return { carry: tail, skippingOversizedLine: true }; + } + return { carry: tail, skippingOversizedLine: false }; + } + + const lineBytes = buf.subarray(start, newlineIdx); + start = newlineIdx + 1; + + if (skipping) { + skipping = false; + continue; + } + + state.lineNumber += 1; + state.totalLineCount += 1; + processLine(lineBytes.toString("utf8"), state.lineNumber, true, state.events, state.seenIds, state.corruptions); + } + + return { carry: Buffer.alloc(0), skippingOversizedLine: skipping }; +} + /** * Replay the JSONL ledger using chunked reads (no whole-file string buffer). * Malformed or partial lines contribute no evidence and are reported as corruption. @@ -116,12 +182,21 @@ export function replayLabLedger(ledgerPath: string): ReplayResult { const fd = openSync(ledgerPath, "r"); const chunkSize = 64 * 1024; const chunk = Buffer.alloc(chunkSize); - let carry = ""; + let carry: Buffer = Buffer.alloc(0); let lineNumber = 0; let totalLineCount = 0; const seenIds = new Set(); let offset = 0; - let hasTrailingNewline = false; + let skippingOversizedLine = false; + + const state = { + lineNumber, + totalLineCount, + skippingOversizedLine, + events, + seenIds, + corruptions, + }; try { while (offset < size) { @@ -129,23 +204,45 @@ export function replayLabLedger(ledgerPath: string): ReplayResult { const n = readSync(fd, chunk, 0, toRead, offset); if (n <= 0) break; offset += n; - carry += chunk.toString("utf8", 0, n); - let idx = carry.indexOf("\n"); - while (idx >= 0) { - const line = carry.slice(0, idx); - carry = carry.slice(idx + 1); - lineNumber += 1; - totalLineCount += 1; - hasTrailingNewline = true; - processLine(line, lineNumber, true, false, events, seenIds, corruptions); - idx = carry.indexOf("\n"); - } + + const combined = carry.length > 0 + ? Buffer.concat([carry, chunk.subarray(0, n)]) + : chunk.subarray(0, n); + const { processable, remainder } = splitIncompleteUtf8Tail(combined); + carry = remainder; + + const result = processBufferedLines(processable, state); + carry = result.carry.length > 0 + ? Buffer.from(Buffer.concat([carry, result.carry])) + : carry; + skippingOversizedLine = result.skippingOversizedLine; + state.skippingOversizedLine = skippingOversizedLine; + lineNumber = state.lineNumber; + totalLineCount = state.totalLineCount; } if (carry.length > 0) { - lineNumber += 1; - totalLineCount += 1; - processLine(carry, lineNumber, hasTrailingNewline, true, events, seenIds, corruptions); + if (skippingOversizedLine) { + lineNumber += 1; + totalLineCount += 1; + corruptions.push({ + kind: "malformed_line", + lineNumber, + detail: `oversized line exceeds ${MAX_SERIALIZED_EVENT_BYTES} bytes`, + }); + } else if (carry.length > MAX_SERIALIZED_EVENT_BYTES) { + lineNumber += 1; + totalLineCount += 1; + corruptions.push({ + kind: "malformed_line", + lineNumber, + detail: `partial line exceeds ${MAX_SERIALIZED_EVENT_BYTES} bytes`, + }); + } else { + lineNumber += 1; + totalLineCount += 1; + processLine(carry.toString("utf8"), lineNumber, false, events, seenIds, corruptions); + } } } finally { closeSync(fd); diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 33fae49c7..165169f1e 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -23,6 +23,7 @@ import type { CaseAuthority, CaseRecord, ScenarioRunResult } from "../conformanc import { expandScenario } from "../conformance/manifest"; import { expandSuiteManifest } from "../conformance/suite-manifest"; import { fixtureDigest } from "../conformance/digest"; +import { resolveProtocolExecutionContext } from "../conformance/executor"; const PACKAGE_VERSION = "2.10.2"; const COMPAT_VERSION = "protocol-v1"; @@ -54,23 +55,34 @@ function behaviorFingerprintForCase(caseRecord: CaseRecord): string { source: "lab_forced", value: upstream, }, + "runtime.arch": { + source: "lab_forced", + value: process.arch, + }, + "runtime.bunVersion": { + source: "lab_forced", + value: process.versions.bun ?? Bun.version, + }, + "runtime.platform": { + source: "lab_forced", + value: process.platform, + }, }, }; return createHash("sha256").update(jcsStringify(values)).digest("hex"); } -function protocolSubject(caseRecord: CaseRecord): ProtocolSubjectV1 { - const inbound = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; - const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; - const surface = caseRecord.requirements.surfaces[0] ?? "responses-http"; +function protocolSubject(caseRecord: CaseRecord, result: ScenarioRunResult): ProtocolSubjectV1 { + const ctx = result.executionContext ?? resolveProtocolExecutionContext(caseRecord); + const upstream = ctx.upstreamProtocol; return { subjectSchemaVersion: 1, subjectKind: "protocol", opencodexCompatibilityVersion: COMPAT_VERSION, effectiveAdapter: upstreamAdapter(upstream), - inboundProtocol: inbound, - upstreamProtocol: upstream, - surface, + inboundProtocol: ctx.inboundProtocol, + upstreamProtocol: ctx.upstreamProtocol, + surface: ctx.surface, behaviorFingerprint: behaviorFingerprintForCase(caseRecord), }; } @@ -108,127 +120,134 @@ export function observationFromConformanceResult( opts: PersistConformanceOptions = {}, ): { event: ObservationEvent; artifacts: ReturnType[] } { const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); - const recordedAt = opts.recordedAt ?? Date.now(); - const startedAt = recordedAt - 1; - const completedAt = recordedAt; - - const expandedScenario = expandScenario(caseRecord, authority); - const scenarioDigest = scenarioManifestDigest(expandedScenario); - const suiteExpanded = expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; - const suiteDigest = suiteManifestDigest(suiteExpanded); - - const fixtureDigests: string[] = []; - const artifacts: ReturnType[] = []; - - const putFixture = (fixture: CaseRecord["fixture"]) => { - const bytes = new TextEncoder().encode(fixture.bytesUtf8); - const digest = fixtureDigest(bytes); - fixtureDigests.push(digest); + try { + const recordedAt = opts.recordedAt ?? Date.now(); + const startedAt = recordedAt - 1; + const completedAt = recordedAt; + + const expandedScenario = expandScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expandedScenario); + const suiteExpanded = expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; + const suiteDigest = suiteManifestDigest(suiteExpanded); + + const fixtureDigests: string[] = []; + const artifacts: ReturnType[] = []; + + const putFixture = (fixture: CaseRecord["fixture"]) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + const digest = fixtureDigest(bytes); + fixtureDigests.push(digest); + artifacts.push( + store.put({ + artifactClass: "fixture", + payload: bytes, + expectedDigest: digest, + mediaType: fixture.mediaType, + }), + ); + }; + putFixture(caseRecord.fixture); + if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); + artifacts.push( store.put({ - artifactClass: "fixture", - payload: bytes, - expectedDigest: digest, - mediaType: fixture.mediaType, + artifactClass: "scenario_manifest", + payload: expandedScenario, + expectedDigest: scenarioDigest, }), ); - }; - putFixture(caseRecord.fixture); - if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); - - artifacts.push( - store.put({ - artifactClass: "scenario_manifest", - payload: expandedScenario, - expectedDigest: scenarioDigest, - }), - ); - artifacts.push( - store.put({ - artifactClass: "suite_manifest", - payload: suiteExpanded, - expectedDigest: suiteDigest, - }), - ); - - const assertionReport = store.put({ - artifactClass: "assertion_report", - payload: { - scenarioId: result.scenarioId, - passed: result.passed, - classification: result.classification, + artifacts.push( + store.put({ + artifactClass: "suite_manifest", + payload: suiteExpanded, + expectedDigest: suiteDigest, + }), + ); + + const assertionReport = store.put({ + artifactClass: "assertion_report", + payload: { + scenarioId: result.scenarioId, + passed: result.passed, + classification: result.classification, + assertions: result.assertionResults.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: a.passed, + observedSummary: a.observedSummary, + reason: a.reason, + })), + }, + }); + artifacts.push(assertionReport); + + const subject = protocolSubject(caseRecord, result); + const subjectId = subjectIdForSubject(subject); + const outcome = outcomeFromResult(result); + const authorityLimits = authority.manifestDefaults.executionLimits; + + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt, + producer: LAB_PRODUCER, + producerVersion: opts.producerVersion ?? PACKAGE_VERSION, + evidenceLayer: "protocol_conformance" as const, + scenarioId: caseRecord.id, + scenarioVersion: String(authority.manifestDefaults.version), + scenarioManifestDigest: scenarioDigest, + suiteId: caseRecord.suite, + suiteVersion: String(authority.manifestDefaults.suiteVersion), + suiteManifestDigest: suiteDigest, + fixtureDigests, + subject, + subjectId, + startedAt, + completedAt, + executionMode: "fixture" as const, + attempt: 1, + limits: { ...authorityLimits }, + outcome, assertions: result.assertionResults.map((a) => ({ id: a.id, operator: a.operator, required: a.required, passed: a.passed, - observedSummary: a.observedSummary, - reason: a.reason, + expectedSummary: "see_assertion_report", + observedSummary: a.observedSummary.slice(0, 512), + ...(a.reason ? { reason: a.reason } : {}), })), - }, - }); - artifacts.push(assertionReport); - - const subject = protocolSubject(caseRecord); - const subjectId = subjectIdForSubject(subject); - const outcome = outcomeFromResult(result); - - const eventWithoutId = { - schemaVersion: LAB_EVENT_SCHEMA_VERSION, - eventKind: "observation" as const, - recordedAt, - producer: LAB_PRODUCER, - producerVersion: opts.producerVersion ?? PACKAGE_VERSION, - evidenceLayer: "protocol_conformance" as const, - scenarioId: caseRecord.id, - scenarioVersion: String(authority.manifestDefaults.version), - scenarioManifestDigest: scenarioDigest, - suiteId: caseRecord.suite, - suiteVersion: String(authority.manifestDefaults.suiteVersion), - suiteManifestDigest: suiteDigest, - fixtureDigests, - subject, - subjectId, - startedAt, - completedAt, - executionMode: "fixture" as const, - attempt: 1, - limits: { ...(authority.manifestDefaults.executionLimits as Record) }, - outcome, - assertions: result.assertionResults.map((a) => ({ - id: a.id, - operator: a.operator, - required: a.required, - passed: a.passed, - expectedSummary: "see_assertion_report", - observedSummary: a.observedSummary.slice(0, 512), - ...(a.reason ? { reason: a.reason } : {}), - })), - environment: { - runtime: { - platform: process.platform, - arch: process.arch, + environment: { + runtime: { + platform: process.platform, + arch: process.arch, + bunVersion: process.versions.bun ?? Bun.version, + }, }, - }, - artifactRefs: artifacts, - ...(result.passed - ? {} - : { - failure: { - class: result.classification, - code: result.secondaryCode ?? result.classification, - retryable: false, - attribution: - result.classification === "harness_failure" - ? ("harness" as const) - : ("opencodex" as const), - }, - }), - }; + artifactRefs: artifacts, + ...(result.passed + ? {} + : { + failure: { + class: result.classification, + code: result.secondaryCode ?? result.classification, + retryable: false, + attribution: + result.classification === "harness_failure" + ? ("harness" as const) + : ("opencodex" as const), + }, + }), + }; - const event = assignEventId(eventWithoutId) as ObservationEvent; - return { event, artifacts }; + const event = assignEventId(eventWithoutId) as ObservationEvent; + return { event, artifacts }; + } finally { + if (ownsStore) store.close(); + } } /** Transform → validate → append one CL-01 result into the canonical JSONL ledger. */ @@ -239,10 +258,18 @@ export function persistConformanceResult( opts: PersistConformanceOptions = {}, ): PersistedConformanceObservation { const paths = ensureLabDirs(opts.configDir); - const { event } = observationFromConformanceResult(result, caseRecord, authority, { - ...opts, - artifactStore: opts.artifactStore ?? createArtifactStore(paths.artifactsDir), - }); - appendLabEvent(paths.ledgerPath, event); - return { event, ledgerPath: paths.ledgerPath }; + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const { event } = observationFromConformanceResult(result, caseRecord, authority, { + ...opts, + artifactStore: store, + }); + appendLabEvent(paths.ledgerPath, event); + return { event, ledgerPath: paths.ledgerPath }; + } finally { + if (ownsStore) store.close(); + } } + +export { behaviorFingerprintForCase }; diff --git a/src/lab/paths.ts b/src/lab/paths.ts index a40befc5d..82f24e282 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -19,15 +19,27 @@ export function labArtifactsDir(configDir = getConfigDir()): string { return join(labRoot(configDir), "artifacts"); } +export function labScratchDir(configDir = getConfigDir()): string { + return join(labRoot(configDir), "scratch"); +} + +export function labExportDir(configDir = getConfigDir()): string { + return join(labRoot(configDir), "export"); +} + /** Ensure lab directories exist with restrictive permissions where the platform allows. */ export function ensureLabDirs(configDir = getConfigDir()): { root: string; ledgerPath: string; sqlitePath: string; artifactsDir: string; + scratchDir: string; + exportDir: string; } { const root = labRoot(configDir); const artifactsDir = labArtifactsDir(configDir); + const scratchDir = labScratchDir(configDir); + const exportDir = labExportDir(configDir); mkdirSync(root, { recursive: true, mode: 0o700 }); mkdirSync(artifactsDir, { recursive: true, mode: 0o700 }); return { @@ -35,5 +47,7 @@ export function ensureLabDirs(configDir = getConfigDir()): { ledgerPath: labLedgerPath(configDir), sqlitePath: labSqlitePath(configDir), artifactsDir, + scratchDir, + exportDir, }; } diff --git a/src/lab/projection/rebuild.ts b/src/lab/projection/rebuild.ts index 778b720e6..6dd2dc7b3 100644 --- a/src/lab/projection/rebuild.ts +++ b/src/lab/projection/rebuild.ts @@ -6,7 +6,7 @@ import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; import { expandScenario, loadCaseAuthority } from "../conformance/manifest"; import { scenarioManifestDigest, jcsStringify } from "../digest"; import { parseSuiteManifestFromArtifact } from "./verification"; -import type { LabEvent, LedgerCorruption } from "../events/types"; +import type { ClaimSnapshotEvent, LabEvent, LedgerCorruption } from "../events/types"; import { loadClaimSourceManifest } from "../artifacts/store"; import { buildInvalidationIndex, isEventExcluded } from "../ledger/invalidation"; import { replayLabLedger } from "../ledger/store"; @@ -81,10 +81,14 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const validation = validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); const authority = loadCaseAuthority(); - const scenarioManifestByDigest = new Map>(); + const scenarioRequirementsByDigest = new Map(); for (const caseRecord of authority.cases) { const expanded = expandScenario(caseRecord, authority); - scenarioManifestByDigest.set(scenarioManifestDigest(expanded), expanded); + scenarioRequirementsByDigest.set(scenarioManifestDigest(expanded), caseRecord.requirements); } const loadSuiteManifest = (digest: string) => { @@ -101,9 +105,10 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const bytes = artifactStore.get(digest); return JSON.parse(new TextDecoder().decode(bytes)) as Record; } catch { - return scenarioManifestByDigest.get(digest) ?? null; + return null; } }; + const loadScenarioRequirements = (digest: string) => scenarioRequirementsByDigest.get(digest) ?? null; const db = new Database(paths.sqlitePath); try { @@ -159,12 +164,24 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const excluded = excludeEventIds(index); const usableClaimEvents = replay.events.filter( - (e) => e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), - ); - const claimStates = resolveClaimStates( - usableClaimEvents.filter((e) => e.eventKind === "claim_snapshot") as never, + (e): e is ClaimSnapshotEvent => + e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), ); + for (const claim of usableClaimEvents) { + const loaded = loadClaimSourceManifest(artifactStore, claim.sourceManifestDigest, { + subjectId: claim.subjectId, + capability: claim.capability, + }); + if (loaded.corruption) { + validation.unusableClaimEventIds.add(claim.eventId); + } + } + + const claimStates = resolveClaimStates(usableClaimEvents, { + unusableClaimEventIds: validation.unusableClaimEventIds, + }); + for (const event of replay.events) { const isExcluded = excluded.has(event.eventId); let exclusionReason: string | null = null; @@ -218,7 +235,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const key = `${event.subjectId}|${event.capability}`; const state = claimStates.states.get(key); const current = state?.current?.eventId === event.eventId ? 1 : 0; - let usable = !isExcluded && !state?.corruption ? 1 : 0; + let usable = !isExcluded && !state?.corruption && !validation.unusableClaimEventIds.has(event.eventId) ? 1 : 0; if (!isExcluded) { const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { @@ -227,7 +244,6 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { }); if (loaded.corruption) { usable = 0; - validation.unusableClaimEventIds.add(event.eventId); corruptions.push({ kind: "claim_corruption", eventId: event.eventId, @@ -295,6 +311,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { unusableClaimEventIds: validation.unusableClaimEventIds, loadSuiteManifest, loadScenarioManifest, + loadScenarioRequirements, }); for (const c of verdictCorruptions) { if (!corruptions.some((x) => x.detail === c.detail && x.eventId === c.eventId)) { diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index 303508a17..fa3dc95a5 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -14,7 +14,7 @@ import { usableObservations, type InvalidationIndex, } from "../ledger/invalidation"; -import { evaluateAllApplicableRequiredPassV1 } from "./verification"; +import { evaluateAllApplicableRequiredPassV1, newestObservationByScenario } from "./verification"; export interface ProjectionKey { subjectId: string; @@ -51,6 +51,7 @@ export interface ClaimState { key: string; current: ClaimSnapshotEvent | null; corruption?: string; + unusable?: boolean; } export interface ProjectVerdictsOptions { @@ -60,16 +61,25 @@ export interface ProjectVerdictsOptions { unusableClaimEventIds?: Set; loadSuiteManifest?: (digest: string) => SuiteManifestV1 | null; loadScenarioManifest?: (digest: string) => Record | null; + loadScenarioRequirements?: (digest: string) => { + inboundProtocols?: string[]; + upstreamProtocols?: string[]; + surfaces?: string[]; + } | null; } /** * Resolve current claims after purge/invalidation and supersession. * Multiple unsuperseded claims, missing predecessors, cross-key supersession, or cycles → UNKNOWN + corruption. */ -export function resolveClaimStates(claims: ClaimSnapshotEvent[]): { +export function resolveClaimStates( + claims: ClaimSnapshotEvent[], + opts: { unusableClaimEventIds?: Set } = {}, +): { states: Map; corruptions: LedgerCorruption[]; } { + const unusableClaims = opts.unusableClaimEventIds ?? new Set(); const byKey = new Map(); for (const claim of claims) { const key = `${claim.subjectId}|${claim.capability}`; @@ -127,7 +137,12 @@ export function resolveClaimStates(claims: ClaimSnapshotEvent[]): { states.set(key, { key, current: null, corruption: "conflicting current claims" }); continue; } - states.set(key, { key, current: current[0] ?? null }); + const currentClaim = current[0] ?? null; + states.set(key, { + key, + current: currentClaim, + unusable: currentClaim ? unusableClaims.has(currentClaim.eventId) : undefined, + }); } return { states, corruptions }; @@ -155,10 +170,10 @@ export function projectVerdicts( const observations = usableObservations(events, index) .filter((o) => o.completedAt <= asOf) .filter((o) => !unusableObs.has(o.eventId)); - const claims = usableClaims(events, index) - .filter((c) => c.effectiveAt <= asOf) - .filter((c) => !unusableClaims.has(c.eventId)); - const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims); + const claims = usableClaims(events, index).filter((c) => c.effectiveAt <= asOf); + const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims, { + unusableClaimEventIds: unusableClaims, + }); corruptions.push(...claimCorruptions); const groups = new Map(); @@ -197,6 +212,7 @@ export function projectVerdicts( verdicts.push( projectObservationGroup(key, ordered, asOf, suiteManifest, { loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, }), ); } @@ -204,7 +220,6 @@ export function projectVerdicts( for (const [, state] of claimStates) { if (!state.current || state.current.polarity !== "supported") continue; if (state.corruption) continue; - if (unusableClaims.has(state.current.eventId)) continue; const claim = state.current; const key: ProjectionKey = { subjectId: claim.subjectId, @@ -216,6 +231,19 @@ export function projectVerdicts( }; const ks = projectionKeyString(key); if (verdicts.some((v) => projectionKeyString(v.key) === ks)) continue; + if (state.unusable) { + verdicts.push({ + key, + verdict: "UNKNOWN", + asOf, + scenarioManifestDigests: [], + claimSourceDigest: claim.sourceManifestDigest, + contributingEventIds: [claim.eventId], + contradictingEventIds: [], + notes: ["current_claim_unusable"], + }); + continue; + } verdicts.push({ key, verdict: "CLAIMED", @@ -236,33 +264,26 @@ function projectObservationGroup( ordered: ObservationEvent[], asOf: number, suiteManifest: SuiteManifestV1 | null, - opts: { loadScenarioManifest?: (digest: string) => Record | null } = {}, + opts: { loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"] } = {}, ): DerivedVerdict { const contributing: string[] = []; const contradicting: string[] = []; const digests = new Set(); const notes: string[] = []; - let sawPass = false; - let sawFail = false; - let sawBlocked = false; - let sawInconclusive = false; - for (const obs of ordered) { digests.add(obs.scenarioManifestDigest); contributing.push(obs.eventId); - if (obs.outcome === "pass") sawPass = true; - else if (obs.outcome === "fail") { - sawFail = true; - contradicting.push(obs.eventId); - } else if (obs.outcome === "blocked") sawBlocked = true; - else if (obs.outcome === "inconclusive") sawInconclusive = true; - else { - const _never: never = obs.outcome; - void _never; - } + if (obs.outcome === "fail") contradicting.push(obs.eventId); } + const newest = newestObservationByScenario(ordered); + const currentObservations = [...newest.values()]; + const currentFails = currentObservations.filter((o) => o.outcome === "fail"); + const currentPasses = currentObservations.filter((o) => o.outcome === "pass"); + const currentBlocked = currentObservations.some((o) => o.outcome === "blocked"); + const currentInconclusive = currentObservations.some((o) => o.outcome === "inconclusive"); + let verdict: CompatibilityVerdict = "UNKNOWN"; if ( key.evidenceLayer !== "protocol_conformance" && @@ -270,18 +291,18 @@ function projectObservationGroup( key.evidenceLayer !== "task_effectiveness" ) { verdict = "UNKNOWN"; - } else if (sawFail && sawPass) { - verdict = "DEGRADED"; - notes.push("contradiction_conservative_v1"); - } else if (sawFail) { - const last = ordered[ordered.length - 1]!; - if (last.failure?.class === "capability_failure" && last.expectedFailure) { + } else if (currentFails.length > 0) { + const lastFail = currentFails.sort((a, b) => { + if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + })[currentFails.length - 1]!; + if (lastFail.failure?.class === "capability_failure" && lastFail.expectedFailure) { verdict = "UNSUPPORTED"; notes.push("capability_absence_control"); } else { verdict = "DEGRADED"; } - } else if (sawPass && !sawInconclusive && !sawBlocked) { + } else if (currentPasses.length > 0 && !currentInconclusive && !currentBlocked) { const executionMode = ordered[0]!.executionMode; const subject = ordered[0]!.subject; if (key.evidenceLayer === "protocol_conformance" && executionMode === "fixture") { @@ -296,14 +317,15 @@ function projectObservationGroup( { subject: subject.subjectKind === "protocol" ? subject : undefined, loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, }, ); notes.push(...evaluation.notes); - if (evaluation.canVerify) { + if (evaluation.applicableRequiredScenarioIds.length === 0) { + verdict = "UNKNOWN"; + } else if (evaluation.canVerify) { verdict = "VERIFIED"; notes.push("all-applicable-required-pass-v1"); - } else if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "PROBED"; } else { verdict = "PROBED"; notes.push("incomplete_required_coverage"); @@ -312,9 +334,9 @@ function projectObservationGroup( } else { verdict = "PROBED"; } - } else if (sawPass) { + } else if (currentPasses.length > 0) { verdict = "PROBED"; - } else if (sawBlocked) { + } else if (currentBlocked) { verdict = "BLOCKED"; } else { verdict = "UNKNOWN"; diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 7af253e61..cef61ec75 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -24,6 +24,20 @@ export function isScenarioApplicable( return true; } +function scenarioApplicableToRequirements( + requirements: { inboundProtocols?: string[]; upstreamProtocols?: string[]; surfaces?: string[] }, + subject: ProtocolSubjectV1, +): boolean { + const inbound = requirements.inboundProtocols ?? []; + const upstream = requirements.upstreamProtocols ?? []; + const surfaces = requirements.surfaces ?? []; + return ( + inbound.includes(subject.inboundProtocol) && + upstream.includes(subject.upstreamProtocol) && + surfaces.includes(subject.surface) + ); +} + function scenarioApplicableToProtocolSubject( scenarioManifest: Record | null, subject: ProtocolSubjectV1, @@ -37,12 +51,11 @@ function scenarioApplicableToProtocolSubject( return ( inbound.includes(subject.inboundProtocol) && upstream.includes(subject.upstreamProtocol) && - surfaces.includes(subject.surface) && - surfaces[0] === subject.surface + surfaces.includes(subject.surface) ); } -function newestObservationByScenario( +export function newestObservationByScenario( observations: ObservationEvent[], ): Map { const byScenario = new Map(); @@ -61,6 +74,12 @@ function newestObservationByScenario( * Positive VERIFIED requires a non-empty applicable required set and a current * pass for every applicable required scenario. */ +export type LoadScenarioRequirements = (digest: string) => { + inboundProtocols?: string[]; + upstreamProtocols?: string[]; + surfaces?: string[]; +} | null; + export function evaluateAllApplicableRequiredPassV1( suiteManifest: SuiteManifestV1, observations: ObservationEvent[], @@ -68,6 +87,7 @@ export function evaluateAllApplicableRequiredPassV1( opts: { subject?: ProtocolSubjectV1; loadScenarioManifest?: LoadScenarioManifest; + loadScenarioRequirements?: LoadScenarioRequirements; } = {}, ): VerificationEvaluation { const notes: string[] = []; @@ -82,17 +102,37 @@ export function evaluateAllApplicableRequiredPassV1( } const requiredScenarios = suiteManifest.scenarios.filter((s) => s.role === "required"); - const applicableRequired = requiredScenarios - .filter((s) => { - if (!isScenarioApplicable(s.id, executionMode, suiteManifest.evidenceLayer)) return false; - if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { - const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; - return scenarioApplicableToProtocolSubject(scenarioManifest, opts.subject); + const applicableRequired: string[] = []; + const unavailableManifests: string[] = []; + + for (const s of requiredScenarios) { + if (!isScenarioApplicable(s.id, executionMode, suiteManifest.evidenceLayer)) continue; + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; + if (scenarioManifest) { + if (!scenarioApplicableToProtocolSubject(scenarioManifest, opts.subject)) continue; + } else { + const requirements = opts.loadScenarioRequirements?.(s.manifestDigest) ?? null; + if (!requirements) { + unavailableManifests.push(s.id); + continue; + } + if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; } - return true; - }) - .map((s) => s.id) - .sort(); + } + applicableRequired.push(s.id); + } + applicableRequired.sort(); + + if (unavailableManifests.length > 0) { + return { + applicableRequiredScenarioIds: applicableRequired, + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [...new Set([...applicableRequired, ...unavailableManifests])].sort(), + canVerify: false, + notes: unavailableManifests.map((id) => `scenario_manifest_unavailable:${id}`), + }; + } if (applicableRequired.length === 0) { return { diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index 23eb7f068..b9aea8501 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -27,11 +27,15 @@ import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PROJECTION_SPEC_VERSION, + behaviorFingerprintForCase, } from "../src/lab"; import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; -import { discoverScenarios, loadCaseAuthority } from "../src/lab/conformance/manifest"; +import { expandSuiteManifest } from "../src/lab/conformance/suite-manifest"; +import { evaluateAllApplicableRequiredPassV1 } from "../src/lab/projection/verification"; +import { discoverScenarios, expandScenario, loadCaseAuthority } from "../src/lab/conformance/manifest"; +import { scenarioManifestDigest } from "../src/lab/digest"; import type { CaseRecord } from "../src/lab/conformance/types"; -import { runScenario } from "../src/lab/conformance/executor"; +import { runScenario, resolveProtocolExecutionContext } from "../src/lab/conformance/executor"; import { LabValidationError } from "../src/lab/events/validate"; import { enforceEventStructureLimits } from "../src/lab/events/limits"; import type { ClaimSnapshotEvent, ObservationEvent, ProtocolSubjectV1 } from "../src/lab/events/types"; @@ -80,6 +84,7 @@ function syntheticPassResult(caseRecord: CaseRecord) { observedSummary: "ok", })), diagnostics: [], + executionContext: resolveProtocolExecutionContext(caseRecord), }; } @@ -140,7 +145,7 @@ function baseObservation(overrides: Partial = {}): Observation completedAt: 1_700_000_000_100, executionMode: "fixture" as const, attempt: 1, - limits: { maxWallClockMs: 1000 }, + limits: { totalTimeoutMs: 1000 }, outcome: "pass" as const, assertions: [ { @@ -152,7 +157,7 @@ function baseObservation(overrides: Partial = {}): Observation observedSummary: "ok", }, ], - environment: { runtime: { platform: "test" } }, + environment: { runtime: { platform: "test", arch: "x64", bunVersion: "1.0.0" } }, artifactRefs: [ { digest: scenarioManifestDigest, @@ -553,7 +558,7 @@ describe("CL-02 projection rebuild determinism", () => { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, ["responses-core"]).filter((c) => c.requirements.upstreamProtocols[0] === "openai-responses" && - c.requirements.surfaces[0] === "responses-sse", + c.requirements.surfaces.includes("responses-sse"), ); let recordedAt = 1_700_000_000_000; for (const caseRecord of scenarios) { @@ -746,7 +751,7 @@ describe("CL-02 review regression coverage", () => { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, ["responses-core"]).filter((c) => c.requirements.upstreamProtocols[0] === "openai-responses" && - c.requirements.surfaces[0] === "responses-sse", + c.requirements.surfaces.includes("responses-sse"), ).slice(0, 2); let t = 1000; const events = scenarios.map((caseRecord) => { @@ -790,22 +795,15 @@ describe("CL-02 review regression coverage", () => { ); const rebuilt = rebuildLabProjection(home); expect((readVerdictSnapshot(rebuilt.sqlitePath)[0] as { verdict: string }).verdict).toBe("PROBED"); - rmSync(join(home, "lab", "artifacts", `${event.scenarioManifestDigest}.bin`)); + const artifactPath = join(home, "lab", "artifacts", `${event.scenarioManifestDigest}.bin`); + expect(existsSync(artifactPath)).toBe(true); + rmSync(artifactPath); const after = rebuildLabProjection(home); expect(readVerdictSnapshot(after.sqlitePath)).toEqual([]); expect(after.corruptions.some((c) => c.kind === "missing_artifact")).toBe(true); }); }); - test("event admission rejects excessive nesting and secret-shaped fields", () => { - let deep: Record = { ok: true }; - for (let i = 0; i < 10; i++) deep = { nested: deep }; - expect(() => enforceEventStructureLimits(deep)).toThrow(); - expect(() => enforceEventStructureLimits({ authorization: "x" })).toThrow(); - const secret = "sk-" + "z".repeat(20); - expect(() => enforceEventStructureLimits({ message: secret })).toThrow(); - }); - test("unusable claim source prevents CLAIMED projection", () => { withHome((home) => { const subject = { @@ -849,3 +847,286 @@ describe("CL-02 review regression coverage", () => { }); }); }); + +describe("CL-02 phase-2 review regressions", () => { + test("putArtifactBytes fails closed on pre-existing symlink target", () => { + withHome((home) => { + const artifacts = join(home, "lab", "artifacts"); + mkdirSync(artifacts, { recursive: true }); + const bytes = new TextEncoder().encode("reuse-me"); + const digest = Bun.CryptoHasher.hash("sha256", bytes, "hex"); + const linkPath = join(artifacts, `${digest}.bin`); + const outside = join(home, "outside.bin"); + writeFileSync(outside, "evil"); + try { + symlinkSync(outside, linkPath); + } catch { + return; + } + const dir = openTrustedArtifactDir(artifacts); + try { + expect(() => putArtifactBytes(dir, bytes)).toThrow(ArtifactFsError); + } finally { + closeTrustedArtifactDir(dir); + } + }); + }); + + test("replay rejects valid line followed by unterminated JSON line", () => { + withHome((home) => { + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + const good = baseObservation(); + const goodLine = `${jcsStringify(good)}\n`; + const partial = jcsStringify(baseObservation({ scenarioId: "responses-core.protocol.request-shape" })); + writeFileSync(ledger, goodLine + partial, "utf8"); + const replay = replayLabLedger(ledger); + expect(replay.events).toHaveLength(1); + expect(replay.corruptions.some((c) => c.kind === "partial_line")).toBe(true); + }); + }); + + test("replay bounds oversized line without newline and continues after newline", () => { + withHome((home) => { + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + const good = baseObservation(); + const huge = "x".repeat(70 * 1024); + writeFileSync(ledger, `${jcsStringify(good)}\n${huge}\n${jcsStringify({ ...good, scenarioId: "responses-core.protocol.sse-framing" })}\n`, "utf8"); + const replay = replayLabLedger(ledger); + expect(replay.events.length).toBeGreaterThanOrEqual(1); + expect(replay.corruptions.some((c) => c.kind === "malformed_line")).toBe(true); + }); + }); + + test("replay handles UTF-8 split across chunk boundaries", () => { + withHome((home) => { + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true }); + const obs = baseObservation({ + assertions: [{ + id: "utf8", + operator: "equals", + required: true, + passed: true, + expectedSummary: "café", + observedSummary: "café", + }], + }); + writeFileSync(ledger, `${jcsStringify(obs)}\n`, "utf8"); + const replay = replayLabLedger(ledger); + expect(replay.events).toHaveLength(1); + expect(replay.corruptions).toHaveLength(0); + }); + }); + + test("zero applicable required scenarios yields UNKNOWN not PROBED", () => { + const authority = loadCaseAuthority(); + const suiteManifest = expandSuiteManifest("responses-core", authority); + const subject = protocolSubject("zero-applicable"); + const loadScenarioManifest = (digest: string) => { + for (const caseRecord of authority.cases) { + const expanded = expandScenario(caseRecord, authority); + if (scenarioManifestDigest(expanded) === digest) return expanded; + } + return null; + }; + const evaluation = evaluateAllApplicableRequiredPassV1( + suiteManifest, + [], + "fixture", + { subject: { ...subject, surface: "responses-ws" }, loadScenarioManifest }, + ); + expect(evaluation.applicableRequiredScenarioIds).toEqual([]); + expect(evaluation.notes).toContain("empty_applicable_required_set"); + const projected = projectVerdicts([]); + expect(projected.verdicts).toEqual([]); + }); + + test("newer pass supersedes older fail without DEGRADED", () => { + const failEvent = assignEventId({ ...baseObservation({ completedAt: 100 }), outcome: "fail" }); + const passEvent = assignEventId({ ...baseObservation({ completedAt: 200 }), outcome: "pass" }); + const projected = projectVerdicts([failEvent, passEvent]); + expect(projected.verdicts[0]?.verdict).not.toBe("DEGRADED"); + expect(projected.verdicts[0]?.contradictingEventIds).toContain(failEvent.eventId); + }); + + test("json-sse-equivalence uses responses-sse execution context", () => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.json-sse-equivalence", + )!; + const ctx = resolveProtocolExecutionContext(caseRecord); + expect(ctx.surface).toBe("responses-sse"); + expect(caseRecord.requirements.surfaces).toContain("responses-http"); + expect(caseRecord.requirements.surfaces).toContain("responses-sse"); + }); + + test("historical manifest missing cannot recover from packaged authority", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"]).find( + (c) => c.id === "responses-core.protocol.sse-framing", + )!; + const { event } = persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: 1000, + }); + rebuildLabProjection(home); + const artifactPath = join(home, "lab", "artifacts", `${event.scenarioManifestDigest}.bin`); + expect(existsSync(artifactPath)).toBe(true); + rmSync(artifactPath); + const after = rebuildLabProjection(home); + expect(readVerdictSnapshot(after.sqlitePath)).toEqual([]); + }); + }); + + test("nested array depth and unknown admission keys fail closed", () => { + let deep: unknown[] = [1]; + for (let i = 0; i < 10; i++) deep = [deep]; + expect(() => enforceEventStructureLimits(deep)).toThrow(); + const obs = baseObservation(); + expect(() => validateLabEvent({ ...obs, limits: { ...obs.limits, rogue: 1 } })).toThrow(); + expect(() => validateLabEvent({ ...obs, assertions: [{ ...obs.assertions[0]!, extra: "x" }] })).toThrow(); + }); + + test("corrupt superseding claim blocks CLAIMED without resurrecting C1", () => { + withHome((home) => { + const subject = { + subjectSchemaVersion: 1 as const, + subjectKind: "route" as const, + providerId: "openai", + providerInstanceFingerprint: createHashHex("inst"), + clientModelId: "gpt", + upstreamModelId: "gpt", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "protocol-v1", + behaviorFingerprint: createHashHex("bf"), + endpointFingerprint: createHashHex("ep"), + dependencies: [], + }; + const subjectId = subjectIdForSubject(subject); + const store = createArtifactStore(join(home, "lab", "artifacts")); + const goodDigest = store.put({ + artifactClass: "claim_source_manifest", + payload: { + schemaVersion: 1, + subjectId, + providerId: "openai", + clientModelId: "gpt", + capability: "tools", + sources: [{ kind: "provider_config", revision: null, facts: { toolCapable: true } }], + resolvedEvidence: { tools: true }, + }, + }).digest; + store.close(); + const c1 = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "claim_snapshot" as const, + recordedAt: 1, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + evidenceLayer: "live_route_compatibility" as const, + subject, + subjectId, + capability: "tools", + polarity: "supported" as const, + sourceManifestDigest: goodDigest, + sourceEventIds: [], + supersedes: [], + effectiveAt: 1, + }) as ClaimSnapshotEvent; + const c2 = assignEventId({ + ...c1, + eventId: undefined as unknown as string, + recordedAt: 2, + effectiveAt: 2, + sourceManifestDigest: createHashHex("missing-manifest"), + supersedes: [c1.eventId], + }) as ClaimSnapshotEvent; + appendLabEvent(join(home, "lab", "compatibility.jsonl"), c1); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), c2); + const rebuilt = rebuildLabProjection(home); + const snap = readVerdictSnapshot(rebuilt.sqlitePath); + expect(snap.every((row) => (row as { verdict: string }).verdict !== "CLAIMED")).toBe(true); + }); + }); + + test("explicit sensitive shared artifact purge removes bytes and dependent events", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, ["responses-core"]).slice(0, 2); + const events = scenarios.map((caseRecord, i) => + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: 1000 + i, + }).event, + ); + const sharedDigest = events[0]!.suiteManifestDigest; + purgeSensitiveEvidence({ + configDir: home, + targetEventIds: [events[0]!.eventId], + targetArtifactDigests: [sharedDigest], + recordedAt: 5000, + }); + const store = createArtifactStore(join(home, "lab", "artifacts")); + try { + expect(() => store.get(sharedDigest)).toThrow(); + } finally { + store.close(); + } + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + expect(replay.events.some((e) => e.eventId === events[0]!.eventId)).toBe(false); + }); + }); + + test("scratch and export purge actions remove bounded lab directories", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"])[0]!; + const { event } = persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: 1000, + }); + const scratch = join(home, "lab", "scratch"); + const exportDir = join(home, "lab", "export"); + mkdirSync(scratch, { recursive: true }); + mkdirSync(exportDir, { recursive: true }); + writeFileSync(join(scratch, "temp.txt"), "x"); + writeFileSync(join(exportDir, "bundle.json"), "{}"); + purgeSensitiveEvidence({ + configDir: home, + targetEventIds: [event.eventId], + targetArtifactDigests: [], + purgeActions: ["ledger", "sqlite", "scratch", "export"], + recordedAt: 5000, + }); + expect(existsSync(join(scratch, "temp.txt"))).toBe(false); + expect(existsSync(join(exportDir, "bundle.json"))).toBe(false); + }); + }); + + test("behavior fingerprint includes frozen runtime keys deterministically", () => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"])[0]!; + const a = behaviorFingerprintForCase(caseRecord); + const b = behaviorFingerprintForCase(caseRecord); + expect(a).toBe(b); + expect(a).toMatch(/^[a-f0-9]{64}$/); + }); + + test("observationFromConformanceResult closes internally created artifact store", () => { + withHome((home) => { + const authority = loadCaseAuthority(); + const caseRecord = discoverScenarios(authority, ["responses-core"])[0]!; + const result = syntheticPassResult(caseRecord); + const { event } = observationFromConformanceResult(result, caseRecord, authority, { configDir: home }); + expect(event.eventKind).toBe("observation"); + const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); + closeTrustedArtifactDir(dir); + }); + }); +}); From 00785fd7d3ea86d9f741ebec5ef2cfd8e43244b5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:13:44 +0200 Subject: [PATCH 07/31] fix(lab): reject symlink targets before artifact create putArtifactBytes and putNamedDigestBytes now probe the digest path with O_NOFOLLOW before writing so rename cannot replace a pre-existing symlink on macOS dirfd artifact I/O. --- src/lab/artifacts/secure-fs.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index 07beb7e98..831c11256 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -270,6 +270,23 @@ function isMissingArtifactError(err: unknown): boolean { return err instanceof ArtifactFsError && err.message.includes("missing"); } +function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): void { + try { + const fd = openAtDir(dir, name, openFlags(O_RDONLY, true)); + try { + const stats = fstatSync(fd); + assertRegularFileStats(stats, "artifact create target"); + harnessFailure("artifact target exists but is not reusable"); + } finally { + closeSync(fd); + } + } catch (err) { + if (isMissingArtifactError(err)) return; + if (err instanceof ArtifactFsError) throw err; + harnessFailure(`artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`); + } +} + function writeTempArtifact( dir: TrustedArtifactDir, tmpName: string, @@ -367,6 +384,7 @@ export function putArtifactBytes( } } + assertArtifactTargetCreatable(dir, digestFileName(digest)); const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; writeTempArtifact(dir, tmpName, bytes, digest, artifactBytesDigest); return readArtifactBytes(dir, digest, bytes.byteLength); @@ -395,6 +413,7 @@ export function putNamedDigestBytes( } } + assertArtifactTargetCreatable(dir, digestFileName(digest)); const tmpName = `.tmp-${digest}-${process.pid}-${Date.now()}.partial`; writeTempArtifact(dir, tmpName, bytes, digest, contentDigest); return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); From afc6697ea9eb615188594bc99c6643cf4a0d007e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:22:12 +0200 Subject: [PATCH 08/31] fix(lab): lstat symlink squatters before artifact create macOS dirfd opens can report ENOENT for symlink digest paths; probe with lstatSync before create so putArtifactBytes fails closed instead of replacing the link. --- src/lab/artifacts/secure-fs.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index 831c11256..23d05c5a4 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -271,17 +271,19 @@ function isMissingArtifactError(err: unknown): boolean { } function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): void { + revalidateDir(dir); + assertRelativeName(name); try { - const fd = openAtDir(dir, name, openFlags(O_RDONLY, true)); - try { - const stats = fstatSync(fd); - assertRegularFileStats(stats, "artifact create target"); - harnessFailure("artifact target exists but is not reusable"); - } finally { - closeSync(fd); + const stats = lstatSync(childPath(dir, name)); + if (stats.isSymbolicLink()) { + harnessFailure("artifact target is a symbolic link"); } + assertRegularFileStats(stats, "artifact create target"); + harnessFailure("artifact target exists but is not reusable"); } catch (err) { - if (isMissingArtifactError(err)) return; + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + return; + } if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`); } From 40725d6970bf92c486a22d01039a9b955effd067 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:10 +0200 Subject: [PATCH 09/31] fix(lab): centralize evidence producer version --- src/lab/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lab/constants.ts b/src/lab/constants.ts index 85f8f3b21..881ca9ea4 100644 --- a/src/lab/constants.ts +++ b/src/lab/constants.ts @@ -3,6 +3,7 @@ export const LAB_EVENT_SCHEMA_VERSION = 1; export const LAB_PROJECTION_SPEC_VERSION = "cl-02.v1"; export const LAB_PRODUCER = "opencodex-lab"; +export const LAB_PRODUCER_VERSION = "2.10.2"; export const MAX_INVALIDATION_TARGETS = 1024; export const MAX_BYTES_PER_ARTIFACT = 256 * 1024; From 1eb173db745c7388d154c20fc6020152635cac1b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:31 +0200 Subject: [PATCH 10/31] fix(lab): fail closed when sanitizing evidence --- src/lab/artifacts/sanitize.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts index 03bd73962..cd9c7e81c 100644 --- a/src/lab/artifacts/sanitize.ts +++ b/src/lab/artifacts/sanitize.ts @@ -9,6 +9,7 @@ import { redactSecretString } from "../../lib/redact"; const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; const SECRETISH = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; +const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { if ( @@ -28,7 +29,9 @@ export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown const FORBIDDEN_CONTRACT_KEYS = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|baseUrl|hostname|account|alias)$/i; function assertNoSecretMaterial(value: unknown, depth: number): void { - if (depth > 8) return; + if (depth > 8) { + throw new Error("contract artifact exceeds sanitization inspection depth"); + } if (typeof value === "string") { if (SECRETISH.test(value)) { throw new Error("contract artifact contains forbidden secret-shaped material"); @@ -78,9 +81,9 @@ function scrubValue(value: unknown, depth: number): unknown { function scrubString(value: string): string { let s = redactSecretString(value); - if (SECRETISH.test(s)) s = s.replace(SECRETISH, "[REDACTED]"); + s = s.replace(SECRETISH_GLOBAL, "[REDACTED]"); // Strip absolute filesystem paths (coarse) - s = s.replace(/(?:[A-Za-z]:\\|\/(?:home|Users|tmp|var|etc)\/)[^\s"']+/g, "[path]"); + s = s.replace(/(?:[A-Za-z]:\\|\/(?:home|Users|tmp|var|etc|root|mnt)\/)[^\s"']+/g, "[path]"); // Strip URL userinfo / private hosts roughly s = s.replace(/https?:\/\/[^\s"']+/gi, (url) => { try { @@ -101,6 +104,11 @@ function scrubString(value: string): string { return s; } +/** Stable privacy boundary for diagnostic text that may be persisted. */ +export function sanitizeDiagnostic(value: unknown): string { + return scrubString(value instanceof Error ? value.message : String(value)); +} + export function sanitizedJsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(scrubValue(value, 0))); } From f7c0a8cf95ca63f5163214692044264afb5ac6b2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:44 +0200 Subject: [PATCH 11/31] fix(lab): enforce restricted state directories --- src/lab/paths.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 82f24e282..d54e11f4d 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,7 +1,14 @@ -import { mkdirSync } from "node:fs"; +import { chmodSync, mkdirSync, statSync } from "node:fs"; import { join } from "node:path"; import { getConfigDir } from "../config"; +function ensureRestrictedDir(dir: string): void { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (process.platform === "win32") return; + const mode = statSync(dir).mode & 0o777; + if (mode !== 0o700) chmodSync(dir, 0o700); +} + /** Canonical Compatibility Lab state root under the OpenCodex config dir. */ export function labRoot(configDir = getConfigDir()): string { return join(configDir, "lab"); @@ -40,8 +47,8 @@ export function ensureLabDirs(configDir = getConfigDir()): { const artifactsDir = labArtifactsDir(configDir); const scratchDir = labScratchDir(configDir); const exportDir = labExportDir(configDir); - mkdirSync(root, { recursive: true, mode: 0o700 }); - mkdirSync(artifactsDir, { recursive: true, mode: 0o700 }); + ensureRestrictedDir(root); + ensureRestrictedDir(artifactsDir); return { root, ledgerPath: labLedgerPath(configDir), From 0afa85265b6f1e222ea4639fef4fe325551329d5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:03 +0200 Subject: [PATCH 12/31] fix(lab): validate suite manifest authority --- src/lab/conformance/suite-manifest.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lab/conformance/suite-manifest.ts b/src/lab/conformance/suite-manifest.ts index 6cab266b9..412a77c84 100644 --- a/src/lab/conformance/suite-manifest.ts +++ b/src/lab/conformance/suite-manifest.ts @@ -34,6 +34,9 @@ export function expandSuiteManifest( } const defaults = authority.manifestDefaults; const capability = cases[0]!.capability; + if (cases.some((caseRecord) => caseRecord.capability !== capability)) { + throw new Error(`suite ${suiteId} declares mixed capabilities`); + } const scenarios: SuiteScenarioRefV1[] = cases .map((caseRecord) => suiteScenarioRef(caseRecord, authority)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); @@ -44,8 +47,8 @@ export function expandSuiteManifest( version: String(defaults.suiteVersion), evidenceLayer: defaults.evidenceLayer, capability, - assertionDslVersion: String(defaults.version), - evidenceSchemaVersion: String(defaults.version), + assertionDslVersion: authority.assertionDslVersion, + evidenceSchemaVersion: authority.evidenceSchemaVersion, freshness: defaults.freshness ?? { maxAgeMs: null }, contradictionRule: "newest-required-observation-v1", scenarios, From 527d97f4cf518630d898beecbf560218241bad81 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:20 +0200 Subject: [PATCH 13/31] fix(lab): bound invalidation target lookup --- src/lab/ledger/invalidation.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/lab/ledger/invalidation.ts b/src/lab/ledger/invalidation.ts index c8b2d317e..5c62b8dca 100644 --- a/src/lab/ledger/invalidation.ts +++ b/src/lab/ledger/invalidation.ts @@ -16,6 +16,8 @@ export interface InvalidationIndex { corruptions: LedgerCorruption[]; } +type EventPosition = { kind: LabEvent["eventKind"]; index: number }; + /** * Apply purge tombstones before ordinary invalidations. * Invalidation target lists are atomic: any bad target rejects the whole event. @@ -27,10 +29,12 @@ export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex { const corruptions: LedgerCorruption[] = []; const validEvidenceIds = new Map(); + const allEventIds = new Map(); - // First pass: record evidence positions; apply purges as encountered. + // First pass: record event positions and evidence positions; apply purges as encountered. for (let i = 0; i < events.length; i++) { const event = events[i]!; + allEventIds.set(event.eventId, { kind: event.eventKind, index: i }); if (event.eventKind === "observation" || event.eventKind === "claim_snapshot") { validEvidenceIds.set(event.eventId, { kind: event.eventKind, index: i }); continue; @@ -45,7 +49,7 @@ export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex { const event = events[i]!; if (event.eventKind !== "invalidation") continue; try { - validateInvalidationTargets(event, events, i, validEvidenceIds, purgedEventIds); + validateInvalidationTargets(event, i, validEvidenceIds, allEventIds); for (const target of event.targetEventIds) { const list = invalidatedBy.get(target) ?? []; list.push(event.eventId); @@ -74,10 +78,9 @@ function applyPurge( function validateInvalidationTargets( event: InvalidationEvent, - all: LabEvent[], index: number, validEvidenceIds: Map, - purgedEventIds: Set, + allEventIds: Map, ): void { for (const target of event.targetEventIds) { if (target === event.eventId) { @@ -85,24 +88,21 @@ function validateInvalidationTargets( } const meta = validEvidenceIds.get(target); if (!meta) { - // Could be unknown, or an invalidation/purge id - const earlier = all.slice(0, index).find((e) => e.eventId === target); - if (!earlier) { - throw new LabValidationError("unknown_target", `unknown target ${target}`); - } - if (earlier.eventKind === "invalidation" || earlier.eventKind === "purge_tombstone") { - throw new LabValidationError("bad_target_kind", `cannot invalidate ${earlier.eventKind}`); + const other = allEventIds.get(target); + if ( + other && + other.index < index && + (other.kind === "invalidation" || other.kind === "purge_tombstone") + ) { + throw new LabValidationError("bad_target_kind", `cannot invalidate ${other.kind}`); } throw new LabValidationError("unknown_target", `unknown target ${target}`); } if (meta.index >= index) { throw new LabValidationError("future_target", `target ${target} is not earlier`); } - // Purged targets: sensitive line may no longer exist — invalidation still must not - // name purge/invalidation kinds; naming a purged observation/claim id is allowed - // only if it appeared earlier as valid evidence before purge. We keep meta from - // first pass so previously-seen evidence ids remain addressable. - void purgedEventIds; + // Purged targets may no longer have a ledger line. A previously valid evidence + // ID remains addressable because validEvidenceIds records its original position. } } From 4f6f9abcce34da862a09795e6b3007d9af190b15 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:37 +0200 Subject: [PATCH 14/31] fix(lab): make purge artifact retention fail closed --- src/lab/ledger/artifact-refs.ts | 55 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/lab/ledger/artifact-refs.ts b/src/lab/ledger/artifact-refs.ts index 8ceba910e..0832f535e 100644 --- a/src/lab/ledger/artifact-refs.ts +++ b/src/lab/ledger/artifact-refs.ts @@ -1,7 +1,7 @@ import type { ClaimSnapshotEvent, LabEvent, ObservationEvent } from "../events/types"; -import { isEventExcluded, type InvalidationIndex } from "./invalidation"; +import type { InvalidationIndex } from "./invalidation"; -/** Collect every artifact digest referenced by non-excluded evidence events. */ +/** Collect artifacts that the frozen CL-00 retention rules still require. */ export function collectReferencedArtifactDigests( events: LabEvent[], index: InvalidationIndex, @@ -12,13 +12,16 @@ export function collectReferencedArtifactDigests( for (const event of events) { if (exclude.has(event.eventId)) continue; - if (isEventExcluded(event.eventId, index)) continue; + if (index.purgedEventIds.has(event.eventId)) continue; if (event.eventKind === "observation") { + // CL-00 releases observation artifacts after invalidation or purge. + if (index.invalidatedBy.has(event.eventId)) continue; addObservationArtifacts(event, refs); continue; } if (event.eventKind === "claim_snapshot") { + // CL-00 retains claim source manifests while any non-purged claim references them. refs.add(event.sourceManifestDigest); } } @@ -47,7 +50,7 @@ export function eventReferencesArtifactDigest(event: LabEvent, digest: string): /** * Expand purge targets so explicitly sensitive artifact digests cannot survive - * while still-referenced evidence remains. + * while any retained ledger evidence line still references them. */ export function expandSensitiveArtifactEventTargets( events: LabEvent[], @@ -58,28 +61,20 @@ export function expandSensitiveArtifactEventTargets( const expanded = new Set(targetEventIds); if (explicitArtifactDigests.size === 0) return expanded; - let changed = true; - while (changed) { - changed = false; - for (const event of events) { - if (expanded.has(event.eventId)) continue; - if (isEventExcluded(event.eventId, index)) continue; - for (const digest of explicitArtifactDigests) { - if (eventReferencesArtifactDigest(event, digest)) { - expanded.add(event.eventId); - changed = true; - break; - } + for (const event of events) { + if (expanded.has(event.eventId)) continue; + if (index.purgedEventIds.has(event.eventId)) continue; + for (const digest of explicitArtifactDigests) { + if (eventReferencesArtifactDigest(event, digest)) { + expanded.add(event.eventId); + break; } } } return expanded; } -/** - * Artifact digests still required by surviving usable evidence after excluding - * the given event IDs (e.g. purge targets). - */ +/** Artifacts still required by surviving evidence after excluding purge targets. */ export function artifactsStillRequired( events: LabEvent[], index: InvalidationIndex, @@ -88,14 +83,19 @@ export function artifactsStillRequired( return collectReferencedArtifactDigests(events, index, { excludeEventIds }); } -/** Digests that may be deleted when purging the given event/artifact targets. */ -export function deletableArtifactDigests( +export interface ArtifactDeletionPlan { + deletable: string[]; + /** Explicitly sensitive digests that remain pinned by surviving evidence. */ + retainedExplicit: string[]; +} + +/** Plan physical artifact deletion for a purge without silently retaining explicit targets. */ +export function artifactDeletionPlan( events: LabEvent[], index: InvalidationIndex, targetEventIds: Set, explicitArtifactDigests: string[], -): string[] { - const explicit = new Set(explicitArtifactDigests); +): ArtifactDeletionPlan { const stillRequired = artifactsStillRequired(events, index, targetEventIds); const candidates = new Set(explicitArtifactDigests); @@ -110,9 +110,10 @@ export function deletableArtifactDigests( } } - return [...candidates] - .filter((digest) => explicit.has(digest) || !stillRequired.has(digest)) - .sort(); + return { + deletable: [...candidates].filter((digest) => !stillRequired.has(digest)).sort(), + retainedExplicit: explicitArtifactDigests.filter((digest) => stillRequired.has(digest)).sort(), + }; } export function observationArtifactDigests(obs: ObservationEvent): string[] { From 21906aced6e06b2f88eb90b43a59256d41e29110 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:54 +0200 Subject: [PATCH 15/31] fix(lab): constrain projection enum columns --- src/lab/projection/schema.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lab/projection/schema.ts b/src/lab/projection/schema.ts index 344b6595d..a8519ad12 100644 --- a/src/lab/projection/schema.ts +++ b/src/lab/projection/schema.ts @@ -2,7 +2,7 @@ * Disposable SQLite projection schema for Compatibility Lab (CL-02). * Not canonical storage — rebuildable from JSONL + content-addressed artifacts. */ -export const LAB_SQLITE_SCHEMA_VERSION = 1; +export const LAB_SQLITE_SCHEMA_VERSION = 2; export const LAB_SQLITE_DDL = ` CREATE TABLE IF NOT EXISTS schema_meta ( @@ -12,12 +12,12 @@ CREATE TABLE IF NOT EXISTS schema_meta ( CREATE TABLE IF NOT EXISTS events ( event_id TEXT PRIMARY KEY, - event_kind TEXT NOT NULL, + event_kind TEXT NOT NULL CHECK (event_kind IN ('observation', 'claim_snapshot', 'invalidation', 'purge_tombstone')), recorded_at INTEGER NOT NULL, producer TEXT NOT NULL, producer_version TEXT NOT NULL, payload_json TEXT NOT NULL, - excluded INTEGER NOT NULL DEFAULT 0, + excluded INTEGER NOT NULL DEFAULT 0 CHECK (excluded IN (0, 1)), exclusion_reason TEXT ); @@ -33,16 +33,16 @@ CREATE TABLE IF NOT EXISTS subjects ( CREATE TABLE IF NOT EXISTS observations ( event_id TEXT PRIMARY KEY, subject_id TEXT NOT NULL, - evidence_layer TEXT NOT NULL, + evidence_layer TEXT NOT NULL CHECK (evidence_layer IN ('protocol_conformance', 'live_route_compatibility', 'task_effectiveness')), suite_id TEXT NOT NULL, suite_version TEXT NOT NULL, suite_manifest_digest TEXT NOT NULL, scenario_id TEXT NOT NULL, scenario_version TEXT NOT NULL, scenario_manifest_digest TEXT NOT NULL, - outcome TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('pass', 'fail', 'blocked', 'inconclusive')), completed_at INTEGER NOT NULL, - execution_mode TEXT NOT NULL, + execution_mode TEXT NOT NULL CHECK (execution_mode IN ('fixture', 'live', 'fabric')), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -54,13 +54,13 @@ CREATE TABLE IF NOT EXISTS claims ( event_id TEXT PRIMARY KEY, subject_id TEXT NOT NULL, capability TEXT NOT NULL, - polarity TEXT NOT NULL, + polarity TEXT NOT NULL CHECK (polarity IN ('supported', 'not_supported', 'withdrawn')), source_manifest_digest TEXT NOT NULL, effective_at INTEGER NOT NULL, recorded_at INTEGER NOT NULL, supersedes_json TEXT NOT NULL, - current INTEGER NOT NULL DEFAULT 0, - usable INTEGER NOT NULL DEFAULT 1, + current INTEGER NOT NULL DEFAULT 0 CHECK (current IN (0, 1)), + usable INTEGER NOT NULL DEFAULT 1 CHECK (usable IN (0, 1)), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -71,7 +71,7 @@ CREATE TABLE IF NOT EXISTS invalidations ( reason TEXT NOT NULL, targets_json TEXT NOT NULL, recorded_at INTEGER NOT NULL, - applied INTEGER NOT NULL DEFAULT 0, + applied INTEGER NOT NULL DEFAULT 0 CHECK (applied IN (0, 1)), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -89,7 +89,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( artifact_class TEXT, media_type TEXT, byte_count INTEGER, - status TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('present', 'corrupt', 'purged_unavailable')), last_error TEXT ); @@ -101,7 +101,7 @@ CREATE TABLE IF NOT EXISTS verdicts ( suite_version TEXT NOT NULL, suite_manifest_digest TEXT NOT NULL, projection_spec_version TEXT NOT NULL, - verdict TEXT NOT NULL, + verdict TEXT NOT NULL CHECK (verdict IN ('UNKNOWN', 'CLAIMED', 'PROBED', 'VERIFIED', 'DEGRADED', 'BLOCKED', 'UNSUPPORTED')), as_of INTEGER NOT NULL, scenario_manifest_digests_json TEXT NOT NULL, claim_source_digest TEXT, From 2af46f3a33df742e11e36773224fe2846d72c985 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:00:35 +0200 Subject: [PATCH 16/31] fix(lab): make sensitive purge progress explicit --- src/lab/ledger/purge.ts | 129 ++++++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 46 deletions(-) diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index ca332fbba..d66662453 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -5,11 +5,16 @@ import { type TrustedArtifactDir, } from "../artifacts/secure-fs"; import { ArtifactFsError } from "../artifacts/secure-fs"; -import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, PURGE_ACTIONS } from "../constants"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PRODUCER_VERSION, + PURGE_ACTIONS, +} from "../constants"; import type { LabEvent, PurgeTombstoneEvent } from "../events/types"; import { assignEventId, validateLabEvent } from "../events/validate"; import { - deletableArtifactDigests, + artifactDeletionPlan, expandSensitiveArtifactEventTargets, } from "./artifact-refs"; import { buildInvalidationIndex } from "./invalidation"; @@ -29,14 +34,16 @@ import { unlinkSync, writeSync, } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; export class PurgeError extends Error { readonly code: string; - constructor(code: string, message: string) { + readonly completedActions: string[]; + constructor(code: string, message: string, completedActions: string[] = []) { super(message); this.name = "PurgeError"; this.code = code; + this.completedActions = [...completedActions]; } } @@ -49,38 +56,61 @@ export interface SensitivePurgeRequest { producerVersion?: string; } +function writeAll(fd: number, bytes: Uint8Array): void { + let offset = 0; + while (offset < bytes.byteLength) { + const n = writeSync(fd, bytes, offset, bytes.byteLength - offset); + if (n <= 0) throw new PurgeError("short_write", "ledger rewrite short write"); + offset += n; + } +} + function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : ""); const bytes = new TextEncoder().encode(body); - const tmpPath = join(join(ledgerPath, ".."), `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); - const fd = openSync(tmpPath, "w", 0o600); + const parent = dirname(ledgerPath); + const tmpPath = join(parent, `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); try { - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) { - throw new PurgeError("short_write", "ledger rewrite short write"); + const fd = openSync(tmpPath, "wx", 0o600); + try { + writeAll(fd, bytes); + fsyncSync(fd); + } finally { + closeSync(fd); } - fsyncSync(fd); - } finally { - closeSync(fd); - } - renameSync(tmpPath, ledgerPath); - const ledgerFd = openSync(ledgerPath, "r+"); - try { - fsyncSync(ledgerFd); - } finally { - closeSync(ledgerFd); + renameSync(tmpPath, ledgerPath); + if (process.platform !== "win32") { + const dirFd = openSync(parent, "r"); + try { + fsyncSync(dirFd); + } finally { + closeSync(dirFd); + } + } + } catch (err) { + try { + unlinkSync(tmpPath); + } catch { + // Preserve the original failure. The temp file may already have been renamed. + } + throw err; } } +function isArtifactMissing(err: unknown): boolean { + return err instanceof ArtifactFsError && ( + err.code === "artifact_missing" || + (err.code === "harness_failure" && err.message.includes("missing")) + ); +} + function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void { const errors: string[] = []; for (const digest of digests) { try { deleteArtifactBytes(dir, digest); } catch (err) { - if (err instanceof ArtifactFsError && err.message.includes("missing")) { - continue; - } + if (isArtifactMissing(err)) continue; errors.push(err instanceof Error ? err.message : String(err)); } } @@ -118,7 +148,7 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto const explicitSensitive = new Set(targetArtifactDigests); const replay = replayLabLedger(paths.ledgerPath); - const index = buildIndexFromReplay(replay.events); + const index = buildInvalidationIndex(replay.events); const removeIds = expandSensitiveArtifactEventTargets( replay.events, index, @@ -131,7 +161,7 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto eventKind: "purge_tombstone" as const, recordedAt: req.recordedAt ?? Date.now(), producer: LAB_PRODUCER, - producerVersion: req.producerVersion ?? "2.10.2", + producerVersion: req.producerVersion ?? LAB_PRODUCER_VERSION, targetEventIds: [...removeIds].sort(), targetArtifactDigests, reason: "sensitive_evidence" as const, @@ -139,33 +169,35 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto }; const tombstone = validateLabEvent(assignEventId(tombstonePayload)) as PurgeTombstoneEvent; - const deletable = purgeActions.includes("artifact") - ? deletableArtifactDigests(replay.events, index, removeIds, targetArtifactDigests) - : []; - - if (purgeActions.includes("artifact") && explicitSensitive.size > 0) { - for (const digest of explicitSensitive) { - if (!deletable.includes(digest)) { - throw new PurgeError( - "sensitive_artifact_not_deletable", - `explicit sensitive artifact ${digest} could not be removed`, - ); - } - } + const deletionPlan = purgeActions.includes("artifact") + ? artifactDeletionPlan(replay.events, index, removeIds, targetArtifactDigests) + : { deletable: [], retainedExplicit: [] }; + + if (deletionPlan.retainedExplicit.length > 0) { + throw new PurgeError( + "sensitive_bytes_retained", + `explicit sensitive artifacts remain required: ${deletionPlan.retainedExplicit.join(",")}`, + ); } let dir: TrustedArtifactDir | null = null; + const completed: string[] = []; try { if (purgeActions.includes("scratch")) { purgeBoundedDirectory(paths.scratchDir); + completed.push("scratch"); } if (purgeActions.includes("export")) { purgeBoundedDirectory(paths.exportDir); + completed.push("export"); } - if (purgeActions.includes("artifact") && deletable.length > 0) { - dir = openTrustedArtifactDir(paths.artifactsDir); - deleteArtifactsFailClosed(dir, deletable); + if (purgeActions.includes("artifact")) { + if (deletionPlan.deletable.length > 0) { + dir = openTrustedArtifactDir(paths.artifactsDir); + deleteArtifactsFailClosed(dir, deletionPlan.deletable); + } + completed.push("artifact"); } if (purgeActions.includes("ledger")) { @@ -179,24 +211,29 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto } else { appendLabEvent(paths.ledgerPath, tombstone); } + completed.push("ledger"); if (purgeActions.includes("sqlite")) { rebuildLabProjection(req.configDir); + completed.push("sqlite"); } return tombstone; } catch (err) { - throw err instanceof PurgeError ? err : new PurgeError("purge_failed", err instanceof Error ? err.message : String(err)); + if (err instanceof PurgeError) { + throw new PurgeError(err.code, err.message, [...completed, ...err.completedActions]); + } + throw new PurgeError( + "purge_failed", + err instanceof Error ? err.message : String(err), + completed, + ); } finally { if (dir) closeTrustedArtifactDir(dir); } } -function buildIndexFromReplay(events: LabEvent[]) { - return buildInvalidationIndex(events); -} - -/** Test helper: read raw ledger text. */ +/** Test helper retained temporarily for compatibility; production callers should replay validated events. */ export function readLedgerText(configDir?: string): string { const paths = ensureLabDirs(configDir); return readFileSync(paths.ledgerPath, "utf8"); From a12a555041ea6427a031fbfc071036487de5111e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:01:04 +0200 Subject: [PATCH 17/31] fix(lab): complete ledger appends across short writes --- src/lab/ledger/store.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 920a58e3e..ade8c3e3d 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -29,9 +29,13 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { const bytes = new TextEncoder().encode(line); const fd = openSync(ledgerPath, "a", 0o600); try { - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) { - throw new LabValidationError("short_write", "ledger append short write"); + let written = 0; + while (written < bytes.byteLength) { + const n = writeSync(fd, bytes, written, bytes.byteLength - written); + if (n <= 0) { + throw new LabValidationError("short_write", "ledger append made no progress"); + } + written += n; } fsyncSync(fd); } finally { From 3dddcdafe751e0f7e54b05aded72727966718ac5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:01:26 +0200 Subject: [PATCH 18/31] fix(lab): reject POSIX paths before ledger admission --- src/lab/events/limits.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lab/events/limits.ts b/src/lab/events/limits.ts index 6fab1fe70..772fc3fba 100644 --- a/src/lab/events/limits.ts +++ b/src/lab/events/limits.ts @@ -75,7 +75,11 @@ export function enforceEventStructureLimits( if (/sk-[a-z0-9]{10,}/i.test(value) || /Bearer\s+\S+/i.test(value)) { throw new LabValidationError("secret_pattern", `${path} contains secret-shaped data`); } - if (/^[A-Za-z]:\\/.test(value) || value.includes("/Users/") || value.includes("\\Users\\")) { + if ( + /^[A-Za-z]:\\/.test(value) || + /(?:^|[\s"'([])\/(?:home|Users|tmp|var|etc|root|mnt)\//.test(value) || + value.includes("\\Users\\") + ) { throw new LabValidationError("raw_path", `${path} contains raw filesystem path`); } return; From f3d878aef3678825b9d1dffd161149c0d8f481a3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:02:03 +0200 Subject: [PATCH 19/31] fix(lab): verify artifact digests by class --- src/lab/artifacts/store.ts | 171 ++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/src/lab/artifacts/store.ts b/src/lab/artifacts/store.ts index 9691e4759..83ad5e118 100644 --- a/src/lab/artifacts/store.ts +++ b/src/lab/artifacts/store.ts @@ -20,7 +20,7 @@ import { readArtifactBytes, type TrustedArtifactDir, } from "./secure-fs"; -import { redactForArtifact } from "./sanitize"; +import { redactForArtifact, sanitizeDiagnostic } from "./sanitize"; export { ArtifactFsError, openTrustedArtifactDir }; export type { TrustedArtifactDir }; @@ -35,11 +35,19 @@ export interface PutArtifactInput { expectedDigest?: string; } +export interface ArtifactReadOptions { + expectedByteCount?: number; + artifactClass?: ArtifactClass; +} + export interface ArtifactStore { dir: TrustedArtifactDir; put(input: PutArtifactInput): ArtifactRefV1; - get(digest: string, expectedByteCount?: number): Uint8Array; - getVerified(digest: string, expectedByteCount?: number): { bytes: Uint8Array; digest: string }; + get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array; + getVerified( + digest: string, + expectedByteCountOrOpts?: number | ArtifactReadOptions, + ): { bytes: Uint8Array; digest: string }; remove(digest: string): void; close(): void; } @@ -50,11 +58,78 @@ function toBytes(payload: Uint8Array | string | unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(payload)); } +function normalizeReadOptions(value?: number | ArtifactReadOptions): ArtifactReadOptions { + return typeof value === "number" ? { expectedByteCount: value } : value ?? {}; +} + +function jsonDigest( + digest: (value: Record) => string, +): (bytes: Uint8Array) => string { + return (bytes) => digest(JSON.parse(new TextDecoder().decode(bytes)) as Record); +} + +function digestForArtifactClass(artifactClass: ArtifactClass): (bytes: Uint8Array) => string { + switch (artifactClass) { + case "fixture": + return fixtureDigest; + case "scenario_manifest": + return jsonDigest(scenarioManifestDigest); + case "suite_manifest": + return jsonDigest(suiteManifestDigest); + case "claim_source_manifest": + return (bytes) => { + const parsed = JSON.parse(new TextDecoder().decode(bytes)); + return claimSourceManifestDigest(validateClaimSourceManifest(parsed).manifest); + }; + default: + return artifactBytesDigest; + } +} + export function createArtifactStore(artifactsDir: string): ArtifactStore { const dir = openTrustedArtifactDir(artifactsDir); let aggregateBytes = 0; let putCount = 0; + const getVerified = ( + digest: string, + expectedByteCountOrOpts?: number | ArtifactReadOptions, + ): { bytes: Uint8Array; digest: string } => { + const opts = normalizeReadOptions(expectedByteCountOrOpts); + const candidates = opts.artifactClass + ? [digestForArtifactClass(opts.artifactClass)] + : [ + artifactBytesDigest, + fixtureDigest, + jsonDigest(scenarioManifestDigest), + jsonDigest(suiteManifestDigest), + digestForArtifactClass("claim_source_manifest"), + ]; + + let lastErr: unknown; + for (const contentDigest of candidates) { + try { + const got = readArtifactBytes(dir, digest, { + expectedByteCount: opts.expectedByteCount, + contentDigest, + }); + return { bytes: got.bytes, digest: got.digest }; + } catch (err) { + lastErr = err; + if ( + err instanceof ArtifactFsError && + err.code !== "artifact_mismatch" && + !err.message.includes("mismatch") + ) { + throw err; + } + } + } + throw lastErr instanceof Error + ? lastErr + : new ArtifactFsError("artifact_mismatch", "artifact digest verification failed"); + }; + return { dir, put(input: PutArtifactInput): ArtifactRefV1 { @@ -79,21 +154,16 @@ export function createArtifactStore(artifactsDir: string): ArtifactStore { let stored; if (isContractClass(input.artifactClass)) { const contractClass = input.artifactClass; - const digest = input.expectedDigest ?? computeContractDigest(contractClass, bytes, redacted); - if (input.expectedDigest && digest !== input.expectedDigest) { - throw new ArtifactFsError("harness_failure", "contract artifact digest mismatch"); - } - const contentDigest = (b: Uint8Array) => - computeContractDigest(contractClass, b, JSON.parse(new TextDecoder().decode(b))); - // Fixtures hash raw bytes; JSON contract manifests hash parsed JCS object. - const hashFn = - contractClass === "fixture" - ? (b: Uint8Array) => fixtureDigest(b) - : contentDigest; - if (hashFn(bytes) !== digest) { - throw new ArtifactFsError("harness_failure", "contract artifact preimage digest mismatch"); + const computedDigest = computeContractDigest(contractClass, bytes, redacted); + if (input.expectedDigest !== undefined && computedDigest !== input.expectedDigest) { + throw new ArtifactFsError("artifact_mismatch", "contract artifact digest mismatch"); } - stored = putNamedDigestBytes(dir, digest, bytes, hashFn); + stored = putNamedDigestBytes( + dir, + computedDigest, + bytes, + digestForArtifactClass(contractClass), + ); } else { stored = putArtifactBytes(dir, bytes, input.expectedDigest); } @@ -109,48 +179,10 @@ export function createArtifactStore(artifactsDir: string): ArtifactStore { artifactClass: input.artifactClass, }; }, - get(digest: string, expectedByteCount?: number): Uint8Array { - return this.getVerified(digest, expectedByteCount).bytes; - }, - getVerified(digest: string, expectedByteCount?: number) { - const candidates: Array<(b: Uint8Array) => string> = [ - artifactBytesDigest, - fixtureDigest, - (b) => { - try { - return scenarioManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - (b) => { - try { - return suiteManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - (b) => { - try { - return claimSourceManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - ]; - let lastErr: unknown; - for (const contentDigest of candidates) { - try { - const got = readArtifactBytes(dir, digest, { expectedByteCount, contentDigest }); - if (got.digest === digest) return { bytes: got.bytes, digest: got.digest }; - } catch (err) { - lastErr = err; - } - } - throw lastErr instanceof Error - ? lastErr - : new ArtifactFsError("harness_failure", "artifact digest verification failed"); + get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array { + return getVerified(digest, expectedByteCountOrOpts).bytes; }, + getVerified, remove(digest: string): void { deleteArtifactBytes(dir, digest); }, @@ -222,24 +254,29 @@ export function putClaimSourceManifest( }); } +export type LoadClaimSourceManifestResult = + | { ok: true; manifest: ClaimSourceManifestV1; corruption?: undefined } + | { ok: false; manifest: ClaimSourceManifestV1 | null; corruption: string }; + export function loadClaimSourceManifest( store: ArtifactStore, digest: string, expected: { subjectId: string; capability: string }, -): { manifest: ClaimSourceManifestV1; corruption?: string } { - if (!isSha256Hex(digest)) return { manifest: null as unknown as ClaimSourceManifestV1, corruption: "invalid digest" }; +): LoadClaimSourceManifestResult { + if (!isSha256Hex(digest)) return { ok: false, manifest: null, corruption: "invalid digest" }; try { - const bytes = store.get(digest); + const bytes = store.get(digest, { artifactClass: "claim_source_manifest" }); const parsed = JSON.parse(new TextDecoder().decode(bytes)); const { manifest, digest: recomputed } = validateClaimSourceManifest(parsed); - if (recomputed !== digest) return { manifest, corruption: "claim-source digest mismatch" }; - if (manifest.subjectId !== expected.subjectId) return { manifest, corruption: "claim-source subjectId mismatch" }; - if (manifest.capability !== expected.capability) return { manifest, corruption: "claim-source capability mismatch" }; - return { manifest }; + if (recomputed !== digest) return { ok: false, manifest, corruption: "claim-source digest mismatch" }; + if (manifest.subjectId !== expected.subjectId) return { ok: false, manifest, corruption: "claim-source subjectId mismatch" }; + if (manifest.capability !== expected.capability) return { ok: false, manifest, corruption: "claim-source capability mismatch" }; + return { ok: true, manifest }; } catch (err) { return { - manifest: null as unknown as ClaimSourceManifestV1, - corruption: err instanceof Error ? err.message : String(err), + ok: false, + manifest: null, + corruption: sanitizeDiagnostic(err), }; } } From 441b78207e6538521905457d3de32c4b9aeb9f67 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:02:56 +0200 Subject: [PATCH 20/31] fix(lab): persist real conformance evidence metadata --- src/lab/observe/from-conformance.ts | 103 ++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 22 deletions(-) diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 165169f1e..44b7e35cb 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -7,6 +7,7 @@ import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, + LAB_PRODUCER_VERSION, type ObservationOutcome, } from "../constants"; import { @@ -19,18 +20,25 @@ import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; import { assignEventId } from "../events/validate"; import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; -import type { CaseAuthority, CaseRecord, ScenarioRunResult } from "../conformance/types"; +import type { + CaseAuthority, + CaseRecord, + ProtocolExecutionContextV1, + ScenarioRunResult, +} from "../conformance/types"; import { expandScenario } from "../conformance/manifest"; -import { expandSuiteManifest } from "../conformance/suite-manifest"; +import { suiteManifestObjectForCase } from "../conformance/suite-manifest"; import { fixtureDigest } from "../conformance/digest"; import { resolveProtocolExecutionContext } from "../conformance/executor"; -const PACKAGE_VERSION = "2.10.2"; const COMPAT_VERSION = "protocol-v1"; export interface PersistConformanceOptions { configDir?: string; recordedAt?: number; + /** Actual execution timestamps from the CL-01 runner; never fabricated. */ + startedAt?: number; + completedAt?: number; producerVersion?: string; artifactStore?: ArtifactStore; } @@ -40,9 +48,33 @@ export interface PersistedConformanceObservation { ledgerPath: string; } -function behaviorFingerprintForCase(caseRecord: CaseRecord): string { - const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; - const adapter = upstreamAdapter(upstream); +function validateExecutionContext( + caseRecord: CaseRecord, + ctx: ProtocolExecutionContextV1, +): ProtocolExecutionContextV1 { + const checks: Array<[string, string[], string]> = [ + ["inboundProtocols", caseRecord.requirements.inboundProtocols, ctx.inboundProtocol], + ["upstreamProtocols", caseRecord.requirements.upstreamProtocols, ctx.upstreamProtocol], + ["surfaces", caseRecord.requirements.surfaces, ctx.surface], + ]; + for (const [name, declared, actual] of checks) { + if (declared.length === 0) throw new Error(`case ${caseRecord.id} has empty ${name}`); + if (!declared.includes(actual)) { + throw new Error(`case ${caseRecord.id} execution context violates ${name}`); + } + } + return ctx; +} + +function behaviorFingerprintForCase( + caseRecord: CaseRecord, + executionContext?: ProtocolExecutionContextV1, +): string { + const ctx = validateExecutionContext( + caseRecord, + executionContext ?? resolveProtocolExecutionContext(caseRecord), + ); + const adapter = upstreamAdapter(ctx.upstreamProtocol); const values = { schemaVersion: 1, resolverVersion: 1, @@ -53,7 +85,7 @@ function behaviorFingerprintForCase(caseRecord: CaseRecord): string { }, "wire.upstreamProtocol": { source: "lab_forced", - value: upstream, + value: ctx.upstreamProtocol, }, "runtime.arch": { source: "lab_forced", @@ -73,17 +105,19 @@ function behaviorFingerprintForCase(caseRecord: CaseRecord): string { } function protocolSubject(caseRecord: CaseRecord, result: ScenarioRunResult): ProtocolSubjectV1 { - const ctx = result.executionContext ?? resolveProtocolExecutionContext(caseRecord); - const upstream = ctx.upstreamProtocol; + const ctx = validateExecutionContext( + caseRecord, + result.executionContext ?? resolveProtocolExecutionContext(caseRecord), + ); return { subjectSchemaVersion: 1, subjectKind: "protocol", opencodexCompatibilityVersion: COMPAT_VERSION, - effectiveAdapter: upstreamAdapter(upstream), + effectiveAdapter: upstreamAdapter(ctx.upstreamProtocol), inboundProtocol: ctx.inboundProtocol, upstreamProtocol: ctx.upstreamProtocol, surface: ctx.surface, - behaviorFingerprint: behaviorFingerprintForCase(caseRecord), + behaviorFingerprint: behaviorFingerprintForCase(caseRecord, ctx), }; } @@ -93,20 +127,43 @@ function upstreamAdapter(protocol: string): string { return "openai-responses"; case "anthropic-messages": return "anthropic"; - default: + case "openai-chat": return "openai-chat"; + default: + throw new Error(`unsupported protocol identity: ${protocol}`); } } function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; - if (result.classification === "timeout" || result.classification === "budget_exhausted") { - return "blocked"; + switch (result.classification) { + case "timeout": + case "budget_exhausted": + return "blocked"; + case "inconclusive": + case "harness_failure": + return "inconclusive"; + case "protocol_failure": + case "capability_failure": + case "behavioral_failure": + return "fail"; + default: { + const _never: never = result.classification; + return _never; + } + } +} + +function requireExecutionTimes(opts: PersistConformanceOptions): { startedAt: number; completedAt: number } { + if (!Number.isInteger(opts.startedAt) || !Number.isInteger(opts.completedAt)) { + throw new Error("real startedAt/completedAt are required for persisted conformance evidence"); } - if (result.classification === "inconclusive" || result.classification === "harness_failure") { - return "inconclusive"; + const startedAt = opts.startedAt!; + const completedAt = opts.completedAt!; + if (startedAt < 0 || completedAt < startedAt) { + throw new Error("invalid persisted conformance execution timestamps"); } - return "fail"; + return { startedAt, completedAt }; } /** @@ -123,13 +180,12 @@ export function observationFromConformanceResult( const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const recordedAt = opts.recordedAt ?? Date.now(); - const startedAt = recordedAt - 1; - const completedAt = recordedAt; + const { startedAt, completedAt } = requireExecutionTimes(opts); + const recordedAt = opts.recordedAt ?? completedAt; const expandedScenario = expandScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); - const suiteExpanded = expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; + const suiteExpanded = suiteManifestObjectForCase(caseRecord, authority); const suiteDigest = suiteManifestDigest(suiteExpanded); const fixtureDigests: string[] = []; @@ -194,7 +250,7 @@ export function observationFromConformanceResult( eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, - producerVersion: opts.producerVersion ?? PACKAGE_VERSION, + producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, evidenceLayer: "protocol_conformance" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), @@ -220,6 +276,9 @@ export function observationFromConformanceResult( observedSummary: a.observedSummary.slice(0, 512), ...(a.reason ? { reason: a.reason } : {}), })), + ...(caseRecord.expectedFailure + ? { expectedFailure: { ...caseRecord.expectedFailure } } + : {}), environment: { runtime: { platform: process.platform, From dca7624aa10ba0529c479855a0a03a9264e26887 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:03:35 +0200 Subject: [PATCH 21/31] fix(lab): enforce verification contracts and freshness --- src/lab/projection/verification.ts | 212 ++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 65 deletions(-) diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index cef61ec75..806a345e0 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,6 +1,8 @@ import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; -import type { ExecutionMode } from "../constants"; +import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; +import type { VerificationRole } from "../conformance/types"; +import { isSha256Hex } from "../digest"; export interface VerificationEvaluation { applicableRequiredScenarioIds: string[]; @@ -12,6 +14,15 @@ export interface VerificationEvaluation { export type LoadScenarioManifest = (digest: string) => Record | null; +export interface ScenarioRequirements { + inboundProtocols?: string[]; + upstreamProtocols?: string[]; + surfaces?: string[]; + freshness?: { maxAgeMs: number | null }; +} + +export type LoadScenarioRequirements = (digest: string) => ScenarioRequirements | null; + /** Live-reserved scenarios are inapplicable in fixture-mode protocol conformance. */ export function isScenarioApplicable( scenarioId: string, @@ -19,13 +30,15 @@ export function isScenarioApplicable( evidenceLayer: string, ): boolean { if (evidenceLayer === "protocol_conformance" && executionMode === "fixture") { - if (scenarioId.includes(".live.")) return false; + // The frozen CL-00 scenario schema has no explicit execution-mode field. + // Use an exact dot-delimited `live` segment rather than substring matching. + if (scenarioId.split(".").includes("live")) return false; } return true; } function scenarioApplicableToRequirements( - requirements: { inboundProtocols?: string[]; upstreamProtocols?: string[]; surfaces?: string[] }, + requirements: ScenarioRequirements, subject: ProtocolSubjectV1, ): boolean { const inbound = requirements.inboundProtocols ?? []; @@ -38,21 +51,46 @@ function scenarioApplicableToRequirements( ); } -function scenarioApplicableToProtocolSubject( +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const maxAgeMs = (value as { maxAgeMs?: unknown }).maxAgeMs; + if (maxAgeMs === null) return { maxAgeMs: null }; + if (!isNonNegativeInteger(maxAgeMs)) return null; + return { maxAgeMs }; +} + +function parseStringArray(value: unknown): string[] | null { + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) return null; + return value; +} + +function scenarioContractFromManifest( scenarioManifest: Record | null, - subject: ProtocolSubjectV1, -): boolean { - if (!scenarioManifest) return false; +): ScenarioRequirements | null { + if (!scenarioManifest) return null; const req = scenarioManifest.requirements; - if (!req || typeof req !== "object") return false; - const inbound = (req as { inboundProtocols?: string[] }).inboundProtocols ?? []; - const upstream = (req as { upstreamProtocols?: string[] }).upstreamProtocols ?? []; - const surfaces = (req as { surfaces?: string[] }).surfaces ?? []; - return ( - inbound.includes(subject.inboundProtocol) && - upstream.includes(subject.upstreamProtocol) && - surfaces.includes(subject.surface) - ); + if (!req || typeof req !== "object" || Array.isArray(req)) return null; + const row = req as Record; + const inboundProtocols = parseStringArray(row.inboundProtocols); + const upstreamProtocols = parseStringArray(row.upstreamProtocols); + const surfaces = parseStringArray(row.surfaces); + if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; + const freshness = parseFreshness(scenarioManifest.freshness); + if (!freshness) return null; + return { inboundProtocols, upstreamProtocols, surfaces, freshness }; +} + +function effectiveMaxAgeMs( + suiteMaxAgeMs: number | null, + scenarioMaxAgeMs: number | null, +): number | null { + if (suiteMaxAgeMs === null) return scenarioMaxAgeMs; + if (scenarioMaxAgeMs === null) return suiteMaxAgeMs; + return Math.min(suiteMaxAgeMs, scenarioMaxAgeMs); } export function newestObservationByScenario( @@ -71,15 +109,9 @@ export function newestObservationByScenario( /** * Evaluate `all-applicable-required-pass-v1` per frozen CL-00 semantics. - * Positive VERIFIED requires a non-empty applicable required set and a current - * pass for every applicable required scenario. + * Positive VERIFIED requires a non-empty applicable required/control set and a + * current, fresh pass for every applicable required scenario and negative control. */ -export type LoadScenarioRequirements = (digest: string) => { - inboundProtocols?: string[]; - upstreamProtocols?: string[]; - surfaces?: string[]; -} | null; - export function evaluateAllApplicableRequiredPassV1( suiteManifest: SuiteManifestV1, observations: ObservationEvent[], @@ -88,6 +120,7 @@ export function evaluateAllApplicableRequiredPassV1( subject?: ProtocolSubjectV1; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; + asOf?: number; } = {}, ): VerificationEvaluation { const notes: string[] = []; @@ -101,26 +134,37 @@ export function evaluateAllApplicableRequiredPassV1( }; } - const requiredScenarios = suiteManifest.scenarios.filter((s) => s.role === "required"); + const requiredScenarios = suiteManifest.scenarios.filter( + (s) => s.role === "required" || s.role === "negative_control", + ); const applicableRequired: string[] = []; const unavailableManifests: string[] = []; + const scenarioMaxAgeById = new Map(); for (const s of requiredScenarios) { if (!isScenarioApplicable(s.id, executionMode, suiteManifest.evidenceLayer)) continue; - if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { - const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; - if (scenarioManifest) { - if (!scenarioApplicableToProtocolSubject(scenarioManifest, opts.subject)) continue; - } else { - const requirements = opts.loadScenarioRequirements?.(s.manifestDigest) ?? null; - if (!requirements) { - unavailableManifests.push(s.id); - continue; - } - if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; + + let requirements: ScenarioRequirements | null = null; + const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; + if (scenarioManifest) { + requirements = scenarioContractFromManifest(scenarioManifest); + if (!requirements) { + unavailableManifests.push(s.id); + continue; + } + } else { + requirements = opts.loadScenarioRequirements?.(s.manifestDigest) ?? null; + if (!requirements || !requirements.freshness) { + unavailableManifests.push(s.id); + continue; } } + + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; + } applicableRequired.push(s.id); + scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } applicableRequired.sort(); @@ -147,6 +191,7 @@ export function evaluateAllApplicableRequiredPassV1( const newest = newestObservationByScenario(observations); const passing: string[] = []; const missing: string[] = []; + const asOf = opts.asOf ?? observations.reduce((max, obs) => Math.max(max, obs.completedAt), 0); for (const scenarioId of applicableRequired) { const scenarioRef = requiredScenarios.find((s) => s.id === scenarioId)!; @@ -160,6 +205,15 @@ export function evaluateAllApplicableRequiredPassV1( notes.push(`digest_mismatch:${scenarioId}`); continue; } + const maxAgeMs = effectiveMaxAgeMs( + suiteManifest.freshness.maxAgeMs, + scenarioMaxAgeById.get(scenarioId) ?? null, + ); + if (maxAgeMs !== null && asOf - obs.completedAt > maxAgeMs) { + missing.push(scenarioId); + notes.push(`stale_observation:${scenarioId}`); + continue; + } if (obs.outcome !== "pass") { missing.push(scenarioId); continue; @@ -176,40 +230,68 @@ export function evaluateAllApplicableRequiredPassV1( }; } +function requireNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + export function parseSuiteManifestFromArtifact(parsed: unknown): SuiteManifestV1 | null { - if (!parsed || typeof parsed !== "object") return null; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const raw = parsed as Record; if (raw.schemaVersion !== 1) return null; - if (typeof raw.id !== "string" || typeof raw.version !== "string") return null; - if (typeof raw.verificationRule !== "string") return null; - if (!Array.isArray(raw.scenarios)) return null; - const scenarios = raw.scenarios.map((s) => { - if (!s || typeof s !== "object") return null; + + const id = requireNonEmptyString(raw.id); + const version = requireNonEmptyString(raw.version); + const capability = requireNonEmptyString(raw.capability); + const assertionDslVersion = requireNonEmptyString(raw.assertionDslVersion); + const evidenceSchemaVersion = requireNonEmptyString(raw.evidenceSchemaVersion); + const contradictionRule = requireNonEmptyString(raw.contradictionRule); + const verificationRule = requireNonEmptyString(raw.verificationRule); + if ( + !id || !version || !capability || !assertionDslVersion || + !evidenceSchemaVersion || !contradictionRule || + verificationRule !== "all-applicable-required-pass-v1" + ) return null; + if ( + typeof raw.evidenceLayer !== "string" || + !(EVIDENCE_LAYERS as readonly string[]).includes(raw.evidenceLayer) + ) return null; + const freshness = parseFreshness(raw.freshness); + if (!freshness || !Array.isArray(raw.scenarios)) return null; + + const roles = new Set(["required", "supplemental", "negative_control"]); + const seenScenarioIds = new Set(); + const scenarios: SuiteManifestV1["scenarios"] = []; + for (const s of raw.scenarios) { + if (!s || typeof s !== "object" || Array.isArray(s)) return null; const row = s as Record; - if (typeof row.id !== "string" || typeof row.version !== "string") return null; - if (typeof row.role !== "string" || typeof row.manifestDigest !== "string") return null; - return { - id: row.id, - version: row.version, - role: row.role as SuiteManifestV1["scenarios"][number]["role"], - manifestDigest: row.manifestDigest, - }; - }); - if (scenarios.some((s) => s === null)) return null; + const scenarioId = requireNonEmptyString(row.id); + const scenarioVersion = requireNonEmptyString(row.version); + const manifestDigest = requireNonEmptyString(row.manifestDigest); + if ( + !scenarioId || !scenarioVersion || !manifestDigest || !isSha256Hex(manifestDigest) || + typeof row.role !== "string" || !roles.has(row.role as VerificationRole) || + seenScenarioIds.has(scenarioId) + ) return null; + seenScenarioIds.add(scenarioId); + scenarios.push({ + id: scenarioId, + version: scenarioVersion, + role: row.role as VerificationRole, + manifestDigest, + }); + } + return { schemaVersion: 1, - id: raw.id, - version: raw.version, - evidenceLayer: String(raw.evidenceLayer ?? ""), - capability: String(raw.capability ?? ""), - assertionDslVersion: String(raw.assertionDslVersion ?? ""), - evidenceSchemaVersion: String(raw.evidenceSchemaVersion ?? ""), - freshness: - raw.freshness && typeof raw.freshness === "object" - ? { maxAgeMs: (raw.freshness as { maxAgeMs?: number | null }).maxAgeMs ?? null } - : { maxAgeMs: null }, - contradictionRule: String(raw.contradictionRule ?? ""), - scenarios: scenarios as SuiteManifestV1["scenarios"], - verificationRule: raw.verificationRule, + id, + version, + evidenceLayer: raw.evidenceLayer, + capability, + assertionDslVersion, + evidenceSchemaVersion, + freshness, + contradictionRule, + scenarios, + verificationRule, }; } From 6ed18383c6b18c56c6128185a416e6b1041c10c2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:04:16 +0200 Subject: [PATCH 22/31] fix(lab): make verdict projection contract-safe --- src/lab/projection/verdicts.ts | 90 +++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index fa3dc95a5..cafe66cd6 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -1,6 +1,7 @@ import type { CompatibilityVerdict } from "../constants"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; +import { jcsStringify } from "../digest"; import type { ClaimSnapshotEvent, LabEvent, @@ -14,7 +15,11 @@ import { usableObservations, type InvalidationIndex, } from "../ledger/invalidation"; -import { evaluateAllApplicableRequiredPassV1, newestObservationByScenario } from "./verification"; +import { + evaluateAllApplicableRequiredPassV1, + newestObservationByScenario, + type ScenarioRequirements, +} from "./verification"; export interface ProjectionKey { subjectId: string; @@ -25,15 +30,23 @@ export interface ProjectionKey { projectionSpecVersion: string; } +function componentKey(parts: readonly string[]): string { + return jcsStringify([...parts]); +} + export function projectionKeyString(key: ProjectionKey): string { - return [ + return componentKey([ key.subjectId, key.evidenceLayer, key.suiteId, key.suiteVersion, key.suiteManifestDigest, key.projectionSpecVersion, - ].join("|"); + ]); +} + +export function claimKeyString(subjectId: string, capability: string): string { + return componentKey([subjectId, capability]); } export interface DerivedVerdict { @@ -61,11 +74,7 @@ export interface ProjectVerdictsOptions { unusableClaimEventIds?: Set; loadSuiteManifest?: (digest: string) => SuiteManifestV1 | null; loadScenarioManifest?: (digest: string) => Record | null; - loadScenarioRequirements?: (digest: string) => { - inboundProtocols?: string[]; - upstreamProtocols?: string[]; - surfaces?: string[]; - } | null; + loadScenarioRequirements?: (digest: string) => ScenarioRequirements | null; } /** @@ -74,15 +83,20 @@ export interface ProjectVerdictsOptions { */ export function resolveClaimStates( claims: ClaimSnapshotEvent[], - opts: { unusableClaimEventIds?: Set } = {}, + opts: { + unusableClaimEventIds?: Set; + purgedEventIds?: ReadonlySet; + } = {}, ): { states: Map; corruptions: LedgerCorruption[]; } { const unusableClaims = opts.unusableClaimEventIds ?? new Set(); + const purgedEventIds = opts.purgedEventIds ?? new Set(); const byKey = new Map(); + const allById = new Map(claims.map((claim) => [claim.eventId, claim])); for (const claim of claims) { - const key = `${claim.subjectId}|${claim.capability}`; + const key = claimKeyString(claim.subjectId, claim.capability); const list = byKey.get(key) ?? []; list.push(claim); byKey.set(key, list); @@ -99,12 +113,16 @@ export function resolveClaimStates( }); const superseded = new Set(); - const byId = new Map(sorted.map((c) => [c.eventId, c])); for (const claim of sorted) { for (const pred of claim.supersedes) { - const prev = byId.get(pred); + const prev = allById.get(pred); if (!prev) { + if (purgedEventIds.has(pred)) { + // A purge may physically remove a superseded predecessor. The ID remains + // valid provenance but is not a live claim candidate. + continue; + } corruptions.push({ kind: "claim_corruption", eventId: claim.eventId, @@ -173,6 +191,7 @@ export function projectVerdicts( const claims = usableClaims(events, index).filter((c) => c.effectiveAt <= asOf); const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims, { unusableClaimEventIds: unusableClaims, + purgedEventIds: index.purgedEventIds, }); corruptions.push(...claimCorruptions); @@ -259,12 +278,25 @@ export function projectVerdicts( return { verdicts, corruptions, index }; } +function isMatchedCapabilityAbsenceControl(obs: ObservationEvent): boolean { + const expected = obs.expectedFailure; + return ( + obs.outcome === "pass" && + !!expected && + expected.controlKind === "capability_absence_control" && + expected.onMatch === "unsupported" + ); +} + function projectObservationGroup( key: ProjectionKey, ordered: ObservationEvent[], asOf: number, suiteManifest: SuiteManifestV1 | null, - opts: { loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"] } = {}, + opts: { + loadScenarioManifest?: (digest: string) => Record | null; + loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + } = {}, ): DerivedVerdict { const contributing: string[] = []; const contradicting: string[] = []; @@ -283,6 +315,12 @@ function projectObservationGroup( const currentPasses = currentObservations.filter((o) => o.outcome === "pass"); const currentBlocked = currentObservations.some((o) => o.outcome === "blocked"); const currentInconclusive = currentObservations.some((o) => o.outcome === "inconclusive"); + const matchedCapabilityAbsence = currentObservations.some(isMatchedCapabilityAbsenceControl); + const currentModes = new Set(currentObservations.map((o) => o.executionMode)); + const newestCurrent = [...currentObservations].sort((a, b) => { + if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }).at(-1); let verdict: CompatibilityVerdict = "UNKNOWN"; if ( @@ -291,21 +329,16 @@ function projectObservationGroup( key.evidenceLayer !== "task_effectiveness" ) { verdict = "UNKNOWN"; + } else if (matchedCapabilityAbsence) { + verdict = "UNSUPPORTED"; + notes.push("capability_absence_control"); } else if (currentFails.length > 0) { - const lastFail = currentFails.sort((a, b) => { - if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; - return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; - })[currentFails.length - 1]!; - if (lastFail.failure?.class === "capability_failure" && lastFail.expectedFailure) { - verdict = "UNSUPPORTED"; - notes.push("capability_absence_control"); - } else { - verdict = "DEGRADED"; - } + verdict = "DEGRADED"; } else if (currentPasses.length > 0 && !currentInconclusive && !currentBlocked) { - const executionMode = ordered[0]!.executionMode; - const subject = ordered[0]!.subject; - if (key.evidenceLayer === "protocol_conformance" && executionMode === "fixture") { + if (currentModes.size > 1) { + verdict = "PROBED"; + notes.push("mixed_execution_modes"); + } else if (key.evidenceLayer === "protocol_conformance" && newestCurrent?.executionMode === "fixture") { if (!suiteManifest) { verdict = "PROBED"; notes.push("suite_manifest_unavailable"); @@ -313,11 +346,12 @@ function projectObservationGroup( const evaluation = evaluateAllApplicableRequiredPassV1( suiteManifest, ordered, - executionMode, + newestCurrent.executionMode, { - subject: subject.subjectKind === "protocol" ? subject : undefined, + subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, loadScenarioManifest: opts.loadScenarioManifest, loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, }, ); notes.push(...evaluation.notes); From 06440007c94118ae45465a85f8beeefc4a6ecd28 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:05:11 +0200 Subject: [PATCH 23/31] fix(lab): rebuild projection atomically --- src/lab/projection/rebuild.ts | 188 ++++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 77 deletions(-) diff --git a/src/lab/projection/rebuild.ts b/src/lab/projection/rebuild.ts index 6dd2dc7b3..bea548167 100644 --- a/src/lab/projection/rebuild.ts +++ b/src/lab/projection/rebuild.ts @@ -1,18 +1,22 @@ import { Database } from "bun:sqlite"; import { existsSync, unlinkSync } from "node:fs"; -import { createArtifactStore } from "../artifacts/store"; +import { createArtifactStore, loadClaimSourceManifest } from "../artifacts/store"; import { ArtifactFsError } from "../artifacts/secure-fs"; +import { sanitizeDiagnostic } from "../artifacts/sanitize"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; import { expandScenario, loadCaseAuthority } from "../conformance/manifest"; import { scenarioManifestDigest, jcsStringify } from "../digest"; -import { parseSuiteManifestFromArtifact } from "./verification"; +import { + parseSuiteManifestFromArtifact, + type ScenarioRequirements, +} from "./verification"; import type { ClaimSnapshotEvent, LabEvent, LedgerCorruption } from "../events/types"; -import { loadClaimSourceManifest } from "../artifacts/store"; import { buildInvalidationIndex, isEventExcluded } from "../ledger/invalidation"; import { replayLabLedger } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import { LAB_SQLITE_DDL, LAB_SQLITE_SCHEMA_VERSION } from "./schema"; import { + claimKeyString, excludeEventIds, projectVerdicts, projectionKeyString, @@ -28,21 +32,27 @@ export interface RebuildResult { function wipeSqlite(path: string): void { for (const candidate of [path, `${path}-wal`, `${path}-shm`]) { + let removed = false; for (let attempt = 0; attempt < 8; attempt++) { try { if (existsSync(candidate)) unlinkSync(candidate); + removed = true; break; } catch (err) { - const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; + const code = err && typeof err === "object" && "code" in err + ? String((err as { code: unknown }).code) + : ""; if (code !== "EBUSY" && code !== "EPERM") throw err; - Bun.sleepSync(20 * (attempt + 1)); + if (attempt < 7) Bun.sleepSync(20 * (attempt + 1)); } } + if (!removed) { + throw new Error(`failed to remove stale projection file after retries: ${candidate}`); + } } } function resetProjectionSchema(db: Database): void { - db.exec("PRAGMA foreign_keys=OFF;"); db.exec(` DROP TABLE IF EXISTS verdicts; DROP TABLE IF EXISTS corruption; @@ -55,7 +65,6 @@ function resetProjectionSchema(db: Database): void { DROP TABLE IF EXISTS events; DROP TABLE IF EXISTS schema_meta; `); - db.exec("PRAGMA foreign_keys=ON;"); db.exec(LAB_SQLITE_DDL); } @@ -81,19 +90,20 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const validation = validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); const authority = loadCaseAuthority(); - const scenarioRequirementsByDigest = new Map(); + const scenarioRequirementsByDigest = new Map(); for (const caseRecord of authority.cases) { const expanded = expandScenario(caseRecord, authority); - scenarioRequirementsByDigest.set(scenarioManifestDigest(expanded), caseRecord.requirements); + scenarioRequirementsByDigest.set(scenarioManifestDigest(expanded), { + inboundProtocols: [...caseRecord.requirements.inboundProtocols], + upstreamProtocols: [...caseRecord.requirements.upstreamProtocols], + surfaces: [...caseRecord.requirements.surfaces], + freshness: { ...authority.manifestDefaults.freshness }, + }); } const loadSuiteManifest = (digest: string) => { try { - const bytes = artifactStore.get(digest); + const bytes = artifactStore.get(digest, { artifactClass: "suite_manifest" }); const parsed = JSON.parse(new TextDecoder().decode(bytes)); return parseSuiteManifestFromArtifact(parsed); } catch { @@ -102,7 +112,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { }; const loadScenarioManifest = (digest: string) => { try { - const bytes = artifactStore.get(digest); + const bytes = artifactStore.get(digest, { artifactClass: "scenario_manifest" }); return JSON.parse(new TextDecoder().decode(bytes)) as Record; } catch { return null; @@ -111,9 +121,14 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const loadScenarioRequirements = (digest: string) => scenarioRequirementsByDigest.get(digest) ?? null; const db = new Database(paths.sqlitePath); + let transactionOpen = false; try { db.exec("PRAGMA journal_mode=DELETE;"); + db.exec("PRAGMA foreign_keys=OFF;"); + db.exec("BEGIN IMMEDIATE;"); + transactionOpen = true; resetProjectionSchema(db); + db.prepare( "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", ).run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION)); @@ -158,8 +173,20 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { VALUES (?, ?, ?, ?, ?)`, ); const insertArtifact = db.prepare( - `INSERT OR REPLACE INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error) - VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(digest) DO UPDATE SET + artifact_class = COALESCE(excluded.artifact_class, artifacts.artifact_class), + media_type = COALESCE(excluded.media_type, artifacts.media_type), + byte_count = COALESCE(excluded.byte_count, artifacts.byte_count), + status = CASE + WHEN artifacts.status = 'purged_unavailable' OR excluded.status = 'purged_unavailable' + THEN 'purged_unavailable' + WHEN artifacts.status = 'corrupt' OR excluded.status = 'corrupt' + THEN 'corrupt' + ELSE 'present' + END, + last_error = COALESCE(excluded.last_error, artifacts.last_error)`, ); const excluded = excludeEventIds(index); @@ -167,19 +194,9 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { (e): e is ClaimSnapshotEvent => e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), ); - - for (const claim of usableClaimEvents) { - const loaded = loadClaimSourceManifest(artifactStore, claim.sourceManifestDigest, { - subjectId: claim.subjectId, - capability: claim.capability, - }); - if (loaded.corruption) { - validation.unusableClaimEventIds.add(claim.eventId); - } - } - const claimStates = resolveClaimStates(usableClaimEvents, { unusableClaimEventIds: validation.unusableClaimEventIds, + purgedEventIds: index.purgedEventIds, }); for (const event of replay.events) { @@ -232,42 +249,24 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { } } else if (event.eventKind === "claim_snapshot") { insertSubject.run(event.subjectId, event.subject.subjectKind, jcsStringify(event.subject)); - const key = `${event.subjectId}|${event.capability}`; + const key = claimKeyString(event.subjectId, event.capability); const state = claimStates.states.get(key); const current = state?.current?.eventId === event.eventId ? 1 : 0; - let usable = !isExcluded && !state?.corruption && !validation.unusableClaimEventIds.has(event.eventId) ? 1 : 0; + const claimCorruption = corruptions.find( + (c) => c.kind === "claim_corruption" && c.eventId === event.eventId, + ); + const usable = !isExcluded && !state?.corruption && + !validation.unusableClaimEventIds.has(event.eventId) ? 1 : 0; if (!isExcluded) { - const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { - subjectId: event.subjectId, - capability: event.capability, - }); - if (loaded.corruption) { - usable = 0; - corruptions.push({ - kind: "claim_corruption", - eventId: event.eventId, - detail: loaded.corruption, - }); - insertCorruption.run("claim_corruption", null, event.eventId, loaded.corruption); - insertArtifact.run( - event.sourceManifestDigest, - "claim_source_manifest", - "application/json", - null, - "corrupt", - loaded.corruption, - ); - } else { - insertArtifact.run( - event.sourceManifestDigest, - "claim_source_manifest", - "application/json", - null, - "present", - null, - ); - } + insertArtifact.run( + event.sourceManifestDigest, + "claim_source_manifest", + "application/json", + null, + claimCorruption ? "corrupt" : "present", + claimCorruption?.detail ?? null, + ); } insertClaim.run( @@ -329,13 +328,6 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { ); for (const v of verdicts) { - if (v.contributingEventIds.every((id) => index.purgedEventIds.has(id))) continue; - if (v.contributingEventIds.some((id) => index.purgedEventIds.has(id) || index.invalidatedBy.has(id))) { - const remaining = v.contributingEventIds.filter( - (id) => !index.purgedEventIds.has(id) && !index.invalidatedBy.has(id), - ); - if (remaining.length === 0) continue; - } insertVerdict.run( projectionKeyString(v.key), v.key.subjectId, @@ -354,13 +346,29 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { ); } + db.exec("COMMIT;"); + transactionOpen = false; return { events: replay.events.length, verdicts: verdicts.length, corruptions, sqlitePath: paths.sqlitePath, }; + } catch (err) { + if (transactionOpen) { + try { + db.exec("ROLLBACK;"); + } catch { + // Preserve the original rebuild failure. + } + } + throw err; } finally { + try { + db.exec("PRAGMA foreign_keys=ON;"); + } catch { + // Closing the disposable DB is still safe if pragma restoration fails. + } db.close(); artifactStore.close(); } @@ -378,13 +386,13 @@ function validateRequiredArtifacts( for (const event of events) { if (isEventExcluded(event.eventId, index)) continue; if (event.eventKind === "observation") { - const required = [ - event.scenarioManifestDigest, - event.suiteManifestDigest, - ...event.fixtureDigests, + const required: Array<{ digest: string; artifactClass: "scenario_manifest" | "suite_manifest" | "fixture" }> = [ + { digest: event.scenarioManifestDigest, artifactClass: "scenario_manifest" }, + { digest: event.suiteManifestDigest, artifactClass: "suite_manifest" }, + ...event.fixtureDigests.map((digest) => ({ digest, artifactClass: "fixture" as const })), ]; let unusable = false; - for (const digest of required) { + for (const { digest, artifactClass } of required) { if (index.purgedArtifactDigests.has(digest)) { corruptions.push({ kind: "missing_artifact", @@ -395,19 +403,46 @@ function validateRequiredArtifacts( continue; } try { - artifactStore.get(digest); + artifactStore.get(digest, { artifactClass }); } catch (err) { + const detail = sanitizeDiagnostic(err); corruptions.push({ - kind: err instanceof ArtifactFsError && err.message.includes("mismatch") + kind: err instanceof ArtifactFsError && + (err.code === "artifact_mismatch" || err.message.includes("mismatch")) ? "artifact_mismatch" : "missing_artifact", eventId: event.eventId, - detail: err instanceof Error ? err.message : String(err), + detail, }); unusable = true; } } if (unusable) unusableObservationIds.add(event.eventId); + continue; + } + + if (event.eventKind === "claim_snapshot") { + if (index.purgedArtifactDigests.has(event.sourceManifestDigest)) { + unusableClaimEventIds.add(event.eventId); + corruptions.push({ + kind: "claim_corruption", + eventId: event.eventId, + detail: `claim source artifact purged: ${event.sourceManifestDigest}`, + }); + continue; + } + const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { + subjectId: event.subjectId, + capability: event.capability, + }); + if (!loaded.ok) { + unusableClaimEventIds.add(event.eventId); + corruptions.push({ + kind: "claim_corruption", + eventId: event.eventId, + detail: loaded.corruption, + }); + } } } @@ -418,7 +453,7 @@ function validateRequiredArtifacts( export function readVerdictSnapshot(sqlitePath: string): unknown[] { const db = new Database(sqlitePath, { readonly: true }); try { - const rows = db + return db .query( `SELECT projection_key, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, projection_spec_version, verdict, @@ -427,7 +462,6 @@ export function readVerdictSnapshot(sqlitePath: string): unknown[] { FROM verdicts ORDER BY projection_key`, ) .all(); - return rows; } finally { db.close(); } From 9ba59cdcf884de7afb5462d4d5315088a34b868d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:06:25 +0200 Subject: [PATCH 24/31] fix(lab): accept measured runner timestamps --- src/lab/observe/from-conformance.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 44b7e35cb..5a165439c 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -33,6 +33,11 @@ import { resolveProtocolExecutionContext } from "../conformance/executor"; const COMPAT_VERSION = "protocol-v1"; +type TimedScenarioRunResult = ScenarioRunResult & { + startedAt?: number; + completedAt?: number; +}; + export interface PersistConformanceOptions { configDir?: string; recordedAt?: number; @@ -154,16 +159,20 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { } } -function requireExecutionTimes(opts: PersistConformanceOptions): { startedAt: number; completedAt: number } { - if (!Number.isInteger(opts.startedAt) || !Number.isInteger(opts.completedAt)) { +function requireExecutionTimes( + result: ScenarioRunResult, + opts: PersistConformanceOptions, +): { startedAt: number; completedAt: number } { + const timed = result as TimedScenarioRunResult; + const startedAt = opts.startedAt ?? timed.startedAt; + const completedAt = opts.completedAt ?? timed.completedAt; + if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt)) { throw new Error("real startedAt/completedAt are required for persisted conformance evidence"); } - const startedAt = opts.startedAt!; - const completedAt = opts.completedAt!; - if (startedAt < 0 || completedAt < startedAt) { + if (startedAt! < 0 || completedAt! < startedAt!) { throw new Error("invalid persisted conformance execution timestamps"); } - return { startedAt, completedAt }; + return { startedAt: startedAt!, completedAt: completedAt! }; } /** @@ -180,7 +189,7 @@ export function observationFromConformanceResult( const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const { startedAt, completedAt } = requireExecutionTimes(opts); + const { startedAt, completedAt } = requireExecutionTimes(result, opts); const recordedAt = opts.recordedAt ?? completedAt; const expandedScenario = expandScenario(caseRecord, authority); From 1ac3d3235d3958c900c6f2f93c7d6cd774ffe81d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:08:58 +0200 Subject: [PATCH 25/31] test(lab): cover phase-2 evidence fixes --- tests/lab-evidence-ledger.test.ts | 163 ++++++++++++++++++------------ 1 file changed, 96 insertions(+), 67 deletions(-) diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index b9aea8501..9678c9dd4 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { chmodSync, existsSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -6,6 +6,7 @@ import { appendLabEvent, assignEventId, buildInvalidationIndex, + claimKeyString, claimSourceManifestDigest, createArtifactStore, eventIdForPayload, @@ -33,7 +34,7 @@ import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDig import { expandSuiteManifest } from "../src/lab/conformance/suite-manifest"; import { evaluateAllApplicableRequiredPassV1 } from "../src/lab/projection/verification"; import { discoverScenarios, expandScenario, loadCaseAuthority } from "../src/lab/conformance/manifest"; -import { scenarioManifestDigest } from "../src/lab/digest"; +import { artifactBytesDigest, scenarioManifestDigest } from "../src/lab/digest"; import type { CaseRecord } from "../src/lab/conformance/types"; import { runScenario, resolveProtocolExecutionContext } from "../src/lab/conformance/executor"; import { LabValidationError } from "../src/lab/events/validate"; @@ -49,10 +50,6 @@ function tempHome(): string { return dir; } -beforeEach(() => { - // OPENCODEX_HOME isolation -}); - afterEach(() => { for (const dir of HOMES.splice(0)) { try { @@ -75,7 +72,7 @@ function syntheticPassResult(caseRecord: CaseRecord) { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure" as const, + classification: "inconclusive" as const, assertionResults: caseRecord.assertions.map((a) => ({ id: a.id, operator: a.operator, @@ -85,6 +82,8 @@ function syntheticPassResult(caseRecord: CaseRecord) { })), diagnostics: [], executionContext: resolveProtocolExecutionContext(caseRecord), + startedAt: 999, + completedAt: 1000, }; } @@ -209,7 +208,6 @@ describe("CL-02 JCS and IDs", () => { const id2 = eventIdForPayload(payload); expect(id1).toBe(id2); expect(isSha256Hex(id1)).toBe(true); - // Domain separation: different domain changes digest expect(id1).not.toBe(subjectId); }); @@ -222,7 +220,6 @@ describe("CL-02 ledger append/replay", () => { test("append and replay round-trip", () => { withHome((home) => { const event = baseObservation(); - // Store required artifacts as opaque named digests for projection const store = createArtifactStore(join(home, "lab", "artifacts")); for (const ref of event.artifactRefs) { store.put({ @@ -231,7 +228,6 @@ describe("CL-02 ledger append/replay", () => { expectedDigest: undefined, }); } - // Write contract-named bytes for digests referenced by the event const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); try { for (const ref of event.artifactRefs) { @@ -322,6 +318,18 @@ describe("CL-02 invalidation validation", () => { }); const index2 = buildInvalidationIndex([obs, bad as never]); expect(index2.corruptions.some((c) => c.kind === "invalid_reference")).toBe(true); + + const invalidatesInvalidation = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: obs.recordedAt + 3, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [inv.eventId], + reason: "manual_correction" as const, + }); + const index3 = buildInvalidationIndex([obs, inv as never, invalidatesInvalidation as never]); + expect(index3.corruptions.some((c) => c.kind === "invalid_reference")).toBe(true); }); }); @@ -395,7 +403,7 @@ describe("CL-02 claim supersession and conflicts", () => { }) as ClaimSnapshotEvent; const ok = resolveClaimStates([c1, c2]); - expect(ok.states.get(`${subjectId}|tools`)?.current?.eventId).toBe(c2.eventId); + expect(ok.states.get(claimKeyString(subjectId, "tools"))?.current?.eventId).toBe(c2.eventId); const c3 = assignEventId({ ...c2, @@ -405,7 +413,7 @@ describe("CL-02 claim supersession and conflicts", () => { supersedes: [], }) as ClaimSnapshotEvent; const conflict = resolveClaimStates([c1, c2, c3]); - expect(conflict.states.get(`${subjectId}|tools`)?.corruption).toBeTruthy(); + expect(conflict.states.get(claimKeyString(subjectId, "tools"))?.corruption).toBeTruthy(); }); test("ClaimSourceManifest rejects secrets and unknown facts", () => { @@ -427,15 +435,19 @@ describe("CL-02 artifacts and secure FS", () => { test("content-addressed put/get and size ceiling", () => { withHome((home) => { const store = createArtifactStore(join(home, "lab", "artifacts")); - const ref = store.put({ artifactClass: "assertion_report", payload: { a: 1 } }); - expect(isSha256Hex(ref.digest)).toBe(true); - const bytes = store.get(ref.digest); - expect(bytes.byteLength).toBe(ref.byteCount); + try { + const ref = store.put({ artifactClass: "assertion_report", payload: { a: 1 } }); + expect(isSha256Hex(ref.digest)).toBe(true); + const bytes = store.get(ref.digest); + expect(bytes.byteLength).toBe(ref.byteCount); - const huge = new Uint8Array(256 * 1024 + 1); - expect(() => - store.put({ artifactClass: "assertion_report", payload: huge }), - ).toThrow(); + const huge = new Uint8Array(256 * 1024 + 1); + expect(() => + store.put({ artifactClass: "assertion_report", payload: huge }), + ).toThrow(); + } finally { + store.close(); + } }); }); @@ -463,7 +475,6 @@ describe("CL-02 artifacts and secure FS", () => { try { symlinkSync(target, linkPath); } catch { - // Windows may require elevation for symlinks return; } const dir = openTrustedArtifactDir(artifacts); @@ -563,12 +574,15 @@ describe("CL-02 projection rebuild determinism", () => { let recordedAt = 1_700_000_000_000; for (const caseRecord of scenarios) { const store = createArtifactStore(join(home, "lab", "artifacts")); - persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { - configDir: home, - recordedAt: recordedAt++, - artifactStore: store, - }); - store.close(); + try { + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: recordedAt++, + artifactStore: store, + }); + } finally { + store.close(); + } } const rebuilt = rebuildLabProjection(home); const snap = readVerdictSnapshot(rebuilt.sqlitePath); @@ -585,13 +599,14 @@ describe("CL-02 projection rebuild determinism", () => { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure", + classification: "inconclusive", assertionResults: [], diagnostics: [], + executionContext: resolveProtocolExecutionContext(caseRecord), }, caseRecord, authority, - { configDir: home, recordedAt: 1000 }, + { configDir: home, recordedAt: 1000, startedAt: 999, completedAt: 1000 }, ); const inv = assignEventId({ schemaVersion: LAB_EVENT_SCHEMA_VERSION, @@ -618,13 +633,14 @@ describe("CL-02 projection rebuild determinism", () => { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure", + classification: "inconclusive", assertionResults: [], diagnostics: [], + executionContext: resolveProtocolExecutionContext(caseRecord), }, caseRecord, authority, - { configDir: home, recordedAt: 1000 }, + { configDir: home, recordedAt: 1000, startedAt: 999, completedAt: 1000 }, ); purgeSensitiveEvidence({ configDir: home, @@ -649,16 +665,22 @@ describe("CL-02 CL-01 integration", () => { const caseRecord = discoverScenarios(authority, ["responses-core"]).find( (c) => c.id === "responses-core.protocol.request-shape", )!; + const startedAt = Date.now(); const result = await runScenario(caseRecord); + const completedAt = Date.now(); expect(result.passed).toBe(true); const { event } = observationFromConformanceResult(result, caseRecord, authority, { configDir: home, - recordedAt: 1_800_000_000_000, + recordedAt: completedAt, + startedAt, + completedAt, }); expect(validateLabEvent(event).eventKind).toBe("observation"); persistConformanceResult(result, caseRecord, authority, { configDir: home, - recordedAt: 1_800_000_000_000, + recordedAt: completedAt, + startedAt, + completedAt, }); const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); expect(replay.validLineCount).toBe(1); @@ -677,21 +699,24 @@ describe("CL-02 privacy canaries", () => { withHome((home) => { const secretCanary = "sk-" + "a".repeat(32); const store = createArtifactStore(join(home, "lab", "artifacts")); - const ref = store.put({ - artifactClass: "error_taxonomy", - payload: { - message: `failed ${secretCanary}`, - path: "C:\\Users\\victim\\secrets\\token.txt", - authorization: "Bearer SUPERSECRET", - url: "https://user:pass@example.com/v1", - }, - }); - const text = new TextDecoder().decode(store.get(ref.digest)); - expect(text).not.toContain(secretCanary); - expect(text).not.toContain("SUPERSECRET"); - expect(text).not.toContain("victim"); - expect(text).not.toContain("user:pass"); - store.close(); + try { + const ref = store.put({ + artifactClass: "error_taxonomy", + payload: { + message: `failed ${secretCanary}`, + path: "C:\\Users\\victim\\secrets\\token.txt", + authorization: "Bearer SUPERSECRET", + url: "https://user:pass@example.com/v1", + }, + }); + const text = new TextDecoder().decode(store.get(ref.digest)); + expect(text).not.toContain(secretCanary); + expect(text).not.toContain("SUPERSECRET"); + expect(text).not.toContain("victim"); + expect(text).not.toContain("user:pass"); + } finally { + store.close(); + } }); }); }); @@ -709,7 +734,6 @@ describe("CL-02 empty/corrupt ledger", () => { }); }); -// Keep chmod import used on POSIX permission smoke (best-effort). void chmodSync; void existsSync; void claimSourceManifestDigest; @@ -728,15 +752,18 @@ describe("CL-02 review regression coverage", () => { const subjectIds = new Set(); for (const caseRecord of scenarios.slice(0, 2)) { const store = createArtifactStore(join(home, "lab", "artifacts")); - const { event } = observationFromConformanceResult( - syntheticPassResult(caseRecord), - caseRecord, - authority, - { configDir: home, recordedAt: t++, artifactStore: store }, - ); - store.close(); - subjectIds.add(event.subjectId); - appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + try { + const { event } = observationFromConformanceResult( + syntheticPassResult(caseRecord), + caseRecord, + authority, + { configDir: home, recordedAt: t++, artifactStore: store }, + ); + subjectIds.add(event.subjectId); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + } finally { + store.close(); + } } expect(subjectIds.size).toBe(1); const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); @@ -756,13 +783,15 @@ describe("CL-02 review regression coverage", () => { let t = 1000; const events = scenarios.map((caseRecord) => { const store = createArtifactStore(join(home, "lab", "artifacts")); - const persisted = persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { - configDir: home, - recordedAt: t++, - artifactStore: store, - }); - store.close(); - return persisted.event; + try { + return persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: t++, + artifactStore: store, + }).event; + } finally { + store.close(); + } }); const sharedDigest = events[0]!.suiteManifestDigest; expect(events[1]!.suiteManifestDigest).toBe(sharedDigest); @@ -854,7 +883,7 @@ describe("CL-02 phase-2 review regressions", () => { const artifacts = join(home, "lab", "artifacts"); mkdirSync(artifacts, { recursive: true }); const bytes = new TextEncoder().encode("reuse-me"); - const digest = Bun.CryptoHasher.hash("sha256", bytes, "hex"); + const digest = artifactBytesDigest(bytes); const linkPath = join(artifacts, `${digest}.bin`); const outside = join(home, "outside.bin"); writeFileSync(outside, "evil"); @@ -1129,4 +1158,4 @@ describe("CL-02 phase-2 review regressions", () => { closeTrustedArtifactDir(dir); }); }); -}); +}); \ No newline at end of file From daab377c228eea7286344f484081e77c13ec3005 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:11:09 +0200 Subject: [PATCH 26/31] fix(lab): classify artifact filesystem failures --- src/lab/artifacts/secure-fs.ts | 98 +++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index 23d05c5a4..421d0669b 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -36,8 +36,8 @@ export class ArtifactFsError extends Error { } } -export function harnessFailure(message: string): never { - throw new ArtifactFsError("harness_failure", message); +export function harnessFailure(message: string, code = "harness_failure"): never { + throw new ArtifactFsError(code, message); } const O_RDONLY = fsConstants.O_RDONLY; @@ -82,16 +82,16 @@ function assertRegularFileStats(stats: Stats, label: string): void { stats.isCharacterDevice() || stats.isBlockDevice() ) { - harnessFailure(`${label}: not a regular file`); + harnessFailure(`${label}: not a regular file`, "artifact_unsafe_target"); } if (stats.nlink !== 1) { - harnessFailure(`${label}: hard links prohibited (nlink=${stats.nlink})`); + harnessFailure(`${label}: hard links prohibited (nlink=${stats.nlink})`, "artifact_unsafe_target"); } } function assertDirectoryStats(stats: Stats, label: string): void { if (!stats.isDirectory() || stats.isSymbolicLink()) { - harnessFailure(`${label}: must be a real directory (no symlink/reparse redirection)`); + harnessFailure(`${label}: must be a real directory (no symlink/reparse redirection)`, "artifact_unsafe_target"); } } @@ -101,7 +101,7 @@ function identityOf(stats: Stats): string { function assertRelativeName(name: string): void { if (name.includes("..") || name.includes("/") || name.includes("\\") || name.includes("\0")) { - harnessFailure("invalid relative artifact name"); + harnessFailure("invalid relative artifact name", "artifact_unsafe_target"); } } @@ -164,7 +164,7 @@ function assertOpenedPathMatchesDescriptor(dir: TrustedArtifactDir, name: string pathEntry.ino !== opened.ino ) { closeSync(fd); - harnessFailure("artifact path identity mismatch after open"); + harnessFailure("artifact path identity mismatch after open", "artifact_unsafe_target"); } } @@ -205,13 +205,13 @@ function revalidateDir(dir: TrustedArtifactDir): void { const stats = fstatSync(dir.fd); assertDirectoryStats(stats, "artifacts dir"); if (identityOf(stats) !== dir.identity) { - harnessFailure("artifacts directory identity changed"); + harnessFailure("artifacts directory identity changed", "artifact_unsafe_target"); } } export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir { const abs = artifactsDir.replace(/[\\/]+$/, ""); - if (abs.includes("\0")) harnessFailure("NUL in artifacts path"); + if (abs.includes("\0")) harnessFailure("NUL in artifacts path", "artifact_unsafe_target"); mkdirSync(abs, { recursive: true, mode: 0o700 }); let fd: number; @@ -219,7 +219,7 @@ export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir try { fd = openSync(abs, openFlags(O_RDONLY | O_DIRECTORY, true)); } catch { - harnessFailure("failed to open artifacts directory with O_DIRECTORY"); + harnessFailure("failed to open artifacts directory with O_DIRECTORY", "artifact_unsafe_target"); } } else { fd = openSync(abs, O_RDONLY); @@ -259,15 +259,16 @@ function readAllFromFd(fd: number, size: number): Buffer { if (n <= 0) break; offset += n; } - if (offset !== size) harnessFailure("short read from artifact descriptor"); + if (offset !== size) harnessFailure("short read from artifact descriptor", "artifact_mismatch"); return buf; } +function isRawMissingError(err: unknown): boolean { + return !!err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT"; +} + function isMissingArtifactError(err: unknown): boolean { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return true; - } - return err instanceof ArtifactFsError && err.message.includes("missing"); + return isRawMissingError(err) || (err instanceof ArtifactFsError && err.code === "artifact_missing"); } function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): void { @@ -276,16 +277,26 @@ function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): v try { const stats = lstatSync(childPath(dir, name)); if (stats.isSymbolicLink()) { - harnessFailure("artifact target is a symbolic link"); + harnessFailure("artifact target is a symbolic link", "artifact_unsafe_target"); } assertRegularFileStats(stats, "artifact create target"); - harnessFailure("artifact target exists but is not reusable"); + harnessFailure("artifact target exists but is not reusable", "artifact_unsafe_target"); } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return; - } + if (isRawMissingError(err)) return; if (err instanceof ArtifactFsError) throw err; - harnessFailure(`artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`); + harnessFailure( + `artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`, + "artifact_unsafe_target", + ); + } +} + +function writeAll(fd: number, bytes: Uint8Array): void { + let offset = 0; + while (offset < bytes.byteLength) { + const n = writeSync(fd, bytes, offset, bytes.byteLength - offset); + if (n <= 0) harnessFailure("artifact write made no progress", "artifact_mismatch"); + offset += n; } } @@ -299,14 +310,17 @@ function writeTempArtifact( let fd: number | null = null; try { fd = openAtDir(dir, tmpName, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) harnessFailure("short write"); + writeAll(fd, bytes); fsyncSync(fd); const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact temp"); - if (stats.size !== bytes.byteLength) harnessFailure("size mismatch after write"); + if (stats.size !== bytes.byteLength) { + harnessFailure("size mismatch after write", "artifact_mismatch"); + } const buf = readAllFromFd(fd, bytes.byteLength); - if (contentDigest(buf) !== digest) harnessFailure("digest mismatch on same descriptor"); + if (contentDigest(buf) !== digest) { + harnessFailure("digest mismatch on same descriptor", "artifact_mismatch"); + } closeSync(fd); fd = null; renameAtDir(dir, tmpName, digestFileName(digest)); @@ -344,16 +358,18 @@ export function readArtifactBytes( const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact fd"); if (opts.expectedByteCount !== undefined && stats.size !== opts.expectedByteCount) { - harnessFailure("artifact size mismatch on descriptor"); + harnessFailure("artifact size mismatch on descriptor", "artifact_mismatch"); + } + if (stats.size > MAX_BYTES_PER_ARTIFACT) { + harnessFailure("artifact exceeds ceiling", "artifact_mismatch"); } - if (stats.size > MAX_BYTES_PER_ARTIFACT) harnessFailure("artifact exceeds ceiling"); const buf = readAllFromFd(fd, stats.size); const got = contentDigest(buf); - if (got !== digest) harnessFailure("artifact digest mismatch on descriptor"); + if (got !== digest) harnessFailure("artifact digest mismatch on descriptor", "artifact_mismatch"); return { digest, bytes: new Uint8Array(buf), byteCount: stats.size }; } catch (err) { - if (isMissingArtifactError(err)) { - harnessFailure(`artifact missing: ${digest}`); + if (isRawMissingError(err)) { + harnessFailure(`artifact missing: ${digest}`, "artifact_missing"); } if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact read failed: ${err instanceof Error ? err.message : String(err)}`); @@ -375,15 +391,15 @@ export function putArtifactBytes( const digest = artifactBytesDigest(bytes); if (expectedDigest !== undefined) { assertDigestName(expectedDigest); - if (digest !== expectedDigest) harnessFailure("artifact digest mismatch before write"); + if (digest !== expectedDigest) { + harnessFailure("artifact digest mismatch before write", "artifact_mismatch"); + } } try { return readArtifactBytes(dir, digest, bytes.byteLength); } catch (err) { - if (!isMissingArtifactError(err)) { - throw err; - } + if (!isMissingArtifactError(err)) throw err; } assertArtifactTargetCreatable(dir, digestFileName(digest)); @@ -404,15 +420,13 @@ export function putNamedDigestBytes( harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); } if (contentDigest(bytes) !== digest) { - harnessFailure("named artifact content digest mismatch before write"); + harnessFailure("named artifact content digest mismatch before write", "artifact_mismatch"); } try { return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); } catch (err) { - if (!isMissingArtifactError(err)) { - throw err; - } + if (!isMissingArtifactError(err)) throw err; } assertArtifactTargetCreatable(dir, digestFileName(digest)); @@ -428,9 +442,7 @@ export function deleteArtifactBytes(dir: TrustedArtifactDir, digest: string): vo try { unlinkAtDir(dir, name); } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return; - } + if (isRawMissingError(err)) return; if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact delete failed: ${err instanceof Error ? err.message : String(err)}`); } @@ -449,9 +461,7 @@ export function artifactExists(dir: TrustedArtifactDir, digest: string): boolean closeSync(fd); } } catch (err) { - if (isMissingArtifactError(err)) { - return false; - } + if (isMissingArtifactError(err)) return false; if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact exists check failed: ${err instanceof Error ? err.message : String(err)}`); } From e5af56aea64f1804bff592c63f4b8f95a7c70ca1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:11:37 +0200 Subject: [PATCH 27/31] fix(lab): remove raw ledger production reader --- src/lab/ledger/purge.ts | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index d66662453..700221b0e 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -28,7 +28,6 @@ import { fsyncSync, openSync, readdirSync, - readFileSync, renameSync, rmSync, unlinkSync, @@ -97,20 +96,13 @@ function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { } } -function isArtifactMissing(err: unknown): boolean { - return err instanceof ArtifactFsError && ( - err.code === "artifact_missing" || - (err.code === "harness_failure" && err.message.includes("missing")) - ); -} - function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void { const errors: string[] = []; for (const digest of digests) { try { deleteArtifactBytes(dir, digest); } catch (err) { - if (isArtifactMissing(err)) continue; + if (err instanceof ArtifactFsError && err.code === "artifact_missing") continue; errors.push(err instanceof Error ? err.message : String(err)); } } @@ -232,9 +224,3 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto if (dir) closeTrustedArtifactDir(dir); } } - -/** Test helper retained temporarily for compatibility; production callers should replay validated events. */ -export function readLedgerText(configDir?: string): string { - const paths = ensureLabDirs(configDir); - return readFileSync(paths.ledgerPath, "utf8"); -} From 02ca5b561dd03db6a371f47a41bb07893f4c02f5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:13:11 +0200 Subject: [PATCH 28/31] docs(lab): document sensitive purge exception --- .../001_pr_stack_status.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 90a9a56de..a7a230879 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (phase-2 review fixes pending push) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (phase-2 review fixes in progress) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its @@ -99,9 +99,13 @@ Independent CL-00 acceptance review is frozen at - **Branch:** `feat/cl-02-evidence-ledger` - **Starting/base SHA:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` (CL-01 merge via #1320) -- **Scope:** immutable JSONL evidence ledger, content-addressed artifact store, - disposable/rebuildable SQLite projection, ClaimSourceManifestV1, invalidation - and sensitive purge tombstones, CL-01 → observation persistence seam. +- **Scope:** append-only JSONL evidence ledger with an explicit sensitive-purge + exception: when the `ledger` purge action is requested, targeted evidence is + physically removed by atomic ledger rewrite and a `purge_tombstone` remains as + the auditable record; SQLite is rebuilt from the rewritten ledger and retained + content-addressed artifacts. The phase also includes the content-addressed + artifact store, disposable/rebuildable SQLite projection, ClaimSourceManifestV1, + invalidation semantics, and the CL-01 → observation persistence seam. - **Explicitly out of scope:** CL-03 live probes, CL-04 CLI/API, CL-05 UI, CL-06 profile fields, Fabric, shadow workflows. @@ -125,10 +129,12 @@ Claims cannot produce `PROBED`/`VERIFIED`. applicability, historical manifest no-substitution, closed event admission, corrupt superseding claims, ArtifactStore lifecycle, frozen behaviour fingerprint). -- **Local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, +- **Previous local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, `tests/lab-evidence-ledger.test.ts` (41/41), `tests/lab-conformance-harness.test.ts` (17/17), `tests/repo-hygiene.test.ts` (11/11), `git diff --check` green on - Windows host. + Windows host before the current CodeRabbit remediation pass. +- **Current CodeRabbit remediation:** committed on draft PR #1333; current CI and + review reconciliation are required before this head may be recorded as accepted. - **Independent acceptance:** not yet — draft PR #1333 remains open for review. - **CL-03:** not started. From eecec3a99571e4985beda5239d9d2fd3c80bad5a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:14:29 +0200 Subject: [PATCH 29/31] refactor(lab): isolate validation error type --- src/lab/events/errors.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/lab/events/errors.ts diff --git a/src/lab/events/errors.ts b/src/lab/events/errors.ts new file mode 100644 index 000000000..17009183a --- /dev/null +++ b/src/lab/events/errors.ts @@ -0,0 +1,9 @@ +export class LabValidationError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(message); + this.name = "LabValidationError"; + this.code = code; + } +} From 0c9dc4d8ec15d16317c9e67edcb8f15cc70ea3b1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:14:45 +0200 Subject: [PATCH 30/31] refactor(lab): break validation import cycle --- src/lab/events/limits.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lab/events/limits.ts b/src/lab/events/limits.ts index 772fc3fba..d6cba306f 100644 --- a/src/lab/events/limits.ts +++ b/src/lab/events/limits.ts @@ -4,7 +4,7 @@ import { MAX_OBJECT_KEYS_PER_EVENT, MAX_SANITIZED_STRING_FIELD, } from "../constants"; -import { LabValidationError } from "./validate"; +import { LabValidationError } from "./errors"; const FORBIDDEN_KEY_RE = /(?:^|_)(?:secret|token|apikey|api_key|password|credential|authorization|cookie|bearer|prompt|repository|filepath|file_path|baseurl|base_url|hostname|rawrequest|raw_request)(?:$|_)/i; From 1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:16:06 +0200 Subject: [PATCH 31/31] fix(lab): validate claim source event ids --- src/lab/events/validate.ts | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index b2db0a0ef..d101e28b0 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -1,4 +1,6 @@ import { enforceEventStructureLimits } from "./limits"; +import { LabValidationError } from "./errors"; +export { LabValidationError } from "./errors"; import { ARTIFACT_CLASSES, ARTIFACT_FILENAME_EXT, @@ -44,15 +46,6 @@ import type { TaskSubjectV1, } from "./types"; -export class LabValidationError extends Error { - readonly code: string; - constructor(code: string, message: string) { - super(message); - this.name = "LabValidationError"; - this.code = code; - } -} - function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -451,9 +444,6 @@ function validateClaimSnapshot(raw: Record): ClaimSnapshotEvent if (!isSha256Hex(sourceManifestDigest)) { throw new LabValidationError("invalid_digest", "sourceManifestDigest"); } - if (!Array.isArray(raw.sourceEventIds) || !Array.isArray(raw.supersedes)) { - throw new LabValidationError("invalid_claim_lists", "sourceEventIds/supersedes"); - } return { schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventId: assertString(raw.eventId, "eventId"), @@ -467,12 +457,14 @@ function validateClaimSnapshot(raw: Record): ClaimSnapshotEvent capability: assertString(raw.capability, "capability"), polarity: assertClosed(raw.polarity, "polarity", CLAIM_POLARITIES), sourceManifestDigest, - sourceEventIds: raw.sourceEventIds.map((id, i) => { - const s = assertString(id, `sourceEventIds[${i}]`); - if (s.length > 0 && !isSha256Hex(s)) throw new LabValidationError("invalid_id", `sourceEventIds[${i}]`); - return s; + sourceEventIds: validateSortedUniqueHexIds(raw.sourceEventIds, "sourceEventIds", { + nonEmpty: false, + max: MAX_INVALIDATION_TARGETS, + }), + supersedes: validateSortedUniqueHexIds(raw.supersedes, "supersedes", { + nonEmpty: false, + max: MAX_INVALIDATION_TARGETS, }), - supersedes: validateSortedUniqueHexIds(raw.supersedes, "supersedes", { nonEmpty: false }), effectiveAt: assertIntMs(raw.effectiveAt, "effectiveAt"), }; }