From 003f7402f49bfe8dd710a7beba52f717051bfadf Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:55:07 +0200 Subject: [PATCH 01/47] feat(lab): implement CL-03 bounded live-route probes Add live manifest authority, sandboxed runner with injectable transport, RouteSubjectV1 builder, live persistence seam, and projection applicability for ten frozen live_route_compatibility scenarios. --- .../001_pr_stack_status.md | 4 +- .../023_live_v1_manifest_authority.md | 56 ++++ .../024_live_v1_cases.json | 176 +++++++++++ .../054_cl03_implementation_record.md | 36 +++ scripts/gen-live-digests.ts | 52 ++++ .../conformance/fixtures/live-v1-cases.json | 176 +++++++++++ src/lab/conformance/types.ts | 17 ++ src/lab/index.ts | 16 + src/lab/live/credential-lease.ts | 59 ++++ src/lab/live/destination.ts | 117 +++++++ src/lab/live/executor.ts | 276 +++++++++++++++++ src/lab/live/inert-tools.ts | 56 ++++ src/lab/live/manifest.ts | 123 ++++++++ src/lab/live/mcp-loopback.ts | 57 ++++ src/lab/live/runner.ts | 32 ++ src/lab/live/sandbox.ts | 108 +++++++ src/lab/live/suite-manifest.ts | 67 ++++ src/lab/live/transport.ts | 144 +++++++++ src/lab/live/types.ts | 103 +++++++ src/lab/observe/from-conformance.ts | 5 + src/lab/observe/from-live.ts | 238 +++++++++++++++ src/lab/paths.ts | 5 + src/lab/projection/verification.ts | 33 +- src/lab/subject/behavior-fingerprint.ts | 33 ++ src/lab/subject/installation-salt.ts | 21 ++ src/lab/subject/route-subject.ts | 67 ++++ tests/lab-live-probe.test.ts | 285 ++++++++++++++++++ tests/lab-live-sandbox.test.ts | 211 +++++++++++++ 28 files changed, 2567 insertions(+), 6 deletions(-) create mode 100644 devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md create mode 100644 devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json create mode 100644 devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md create mode 100644 scripts/gen-live-digests.ts create mode 100644 src/lab/conformance/fixtures/live-v1-cases.json create mode 100644 src/lab/live/credential-lease.ts create mode 100644 src/lab/live/destination.ts create mode 100644 src/lab/live/executor.ts create mode 100644 src/lab/live/inert-tools.ts create mode 100644 src/lab/live/manifest.ts create mode 100644 src/lab/live/mcp-loopback.ts create mode 100644 src/lab/live/runner.ts create mode 100644 src/lab/live/sandbox.ts create mode 100644 src/lab/live/suite-manifest.ts create mode 100644 src/lab/live/transport.ts create mode 100644 src/lab/live/types.ts create mode 100644 src/lab/observe/from-live.ts create mode 100644 src/lab/subject/behavior-fingerprint.ts create mode 100644 src/lab/subject/installation-salt.ts create mode 100644 src/lab/subject/route-subject.ts create mode 100644 tests/lab-live-probe.test.ts create mode 100644 tests/lab-live-sandbox.test.ts diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 95f58deac..39e7adf73 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -22,7 +22,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | | CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | NOT RECORDED | [#1333](https://github.com/lidge-jun/opencodex/pull/1333) | MERGED TO `dev` at `025c37916225dd685d9217e5b40190600f06d278`; POST-MERGE HARDENING [#1343](https://github.com/lidge-jun/opencodex/pull/1343) MERGED at `eee2dab4d1bbacefce56057adad51d734f346702`; FINAL CLOSURE GATE [#1348](https://github.com/lidge-jun/opencodex/pull/1348) | -| CL-03 | — | — | — | — | NOT STARTED; AUTHORIZATION ACTIVATES ON #1348 MERGE AFTER GREEN CI/REVIEW | +| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | — | — | IN PROGRESS (implementation) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -156,4 +156,4 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-00: **ACCEPTED** (merged #1286). - CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. - CL-02: **MERGED** via #1333 at `025c37916225dd685d9217e5b40190600f06d278`; post-merge hardening #1343 is also **MERGED** at `eee2dab4d1bbacefce56057adad51d734f346702`; final closure is tracked in #1348. -- CL-03: **NOT STARTED**. Authorization is activated by merge of #1348 after green required CI and zero unresolved valid CodeRabbit findings. +- CL-03: **IN PROGRESS** on `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`. diff --git a/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md b/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md new file mode 100644 index 000000000..0e599345e --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md @@ -0,0 +1,56 @@ +# CL-03 live V1 manifest authority + +This document closes the executable semantics for the initial +`live_route_compatibility` scenarios. It is normative for CL-03. + +The machine-readable source of truth is +[`024_live_v1_cases.json`](./024_live_v1_cases.json). It contains 10 frozen +live-route scenarios with Lab-authored synthetic fixtures, literal expected +values, row-specific requirements, execution limits from the security contract, +artifact policy, failure rules, and domain-separated fixture digests. + +## 1. Manifest expansion + +For each entry in `cases`, CL-03 constructs `CompatibilityScenarioV1` using the +same `fixtureRef` contract as protocol V1, with authority +`024_live_v1_cases.json`. Defaults include: + +- `evidenceLayer`: `live_route_compatibility` +- `executionMode`: `live` +- `freshness.maxAgeMs`: `604800000` (7 days) +- `failureRuleSet`: `live-v1-default` + +Environmental blockers (`authentication_blocked`, `quota_blocked`, +`network_failure`, `provider_transient`, `region_blocked`, `timeout`, +`budget_exhausted`) map to `verdictEffect: none` and never produce compatibility +degradation. + +## 2. Live suites + +| Suite | Live scenario | +|---|---| +| `responses-core` | `responses-core.live.basic-turn` | +| `chat-core` | `chat-core.live.basic-turn` | +| `anthropic-core` | `anthropic-core.live.basic-turn` | +| `tools-core` | `tools-core.live.function-round-trip`, `tools-core.live.custom-freeform-round-trip` | +| `codex-core` | `codex-core.live.tool-turn`, `codex-core.live.custom-tool-turn` | +| `vision-core` | `vision-core.live.synthetic-ocr` | +| `reasoning-core` | `reasoning-core.live.replay` | +| `mcp-core` | `mcp-core.live.synthetic-tool` | + +## 3. Route subject and sandbox + +Live evidence uses `RouteSubjectV1` with opaque `endpointFingerprint` and +`providerInstanceFingerprint` derived via `localFingerprint()` from immutable +destination snapshots. Raw URLs, credentials, and DNS results are never +persisted. + +The live sandbox rejects proxy environment variables, permits only +`TZ=UTC` and `NO_COLOR=1`, enforces security-contract resource ceilings, and +uses injectable transport in tests. + +## 4. CL-03 boundary + +CL-03 implements the live runner, sandbox, route subject builder, persistence +seam, and projection applicability for `live_route_compatibility`. It does not +implement CL-04 CLI/API. diff --git a/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json new file mode 100644 index 000000000..ded71e84a --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json @@ -0,0 +1,176 @@ +{ + "schemaVersion": 1, + "authority": "CL-03 live manifest authority", + "sourceCommit": "4f746d13799888ea0a8c7a111aa2ad61c2126ea0", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "live-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "auth-blocked", "match": ["auth_blocked", "authentication_blocked"], "classification": "authentication_blocked", "secondaryCode": "auth_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "quota-blocked", "match": ["quota_blocked"], "classification": "quota_blocked", "secondaryCode": "quota_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "network-blocked", "match": ["network_blocked", "network_failure"], "classification": "network_failure", "secondaryCode": "network_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "region-blocked", "match": ["region_blocked"], "classification": "region_blocked", "secondaryCode": "region_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "transient-blocked", "match": ["provider_transient"], "classification": "provider_transient", "secondaryCode": "provider_transient", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "live_route_compatibility", + "verificationRole": "required", + "executionMode": "live", + "freshness": { "maxAgeMs": 604800000 }, + "executionLimits": { + "totalTimeoutMs": 120000, + "connectTimeoutMs": 10000, + "firstByteTimeoutMs": 30000, + "inactivityTimeoutMs": 30000, + "maxRequests": 16, + "maxInputBytes": 8388608, + "maxOutputBytes": 16777216, + "maxOutputTokens": 32768, + "maxToolCalls": 32, + "maxMemoryBytes": 536870912, + "maxChildProcesses": 0, + "maxArtifacts": 16, + "perArtifactBytes": 262144, + "aggregateArtifactBytes": 1048576, + "maxArtifactBytes": 262144 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_live_v1" + }, + "failureRuleSet": "live-v1-default" + }, + "cases": [ + { + "id": "responses-core.live.basic-turn", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "rsp-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "rsp-live-basic-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]}", "digest": "8008bc78564ccf7676fba34035df94bf3f342f186ad7778f75efa71058604429" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.live.basic-turn", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "chat-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.live.basic-turn", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "anthropic-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "tools-core.live.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-fn-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"function\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"tool_choice\":{\"type\":\"function\",\"name\":\"lookup\"}}", "digest": "50b83cb763832dee3c68def08fd478c40eaf52ee2f653392e9a42935eea4a682" }, + "fixture": { "id": "tools-live-fn-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"function_call\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}]}", "digest": "18f02e77d01a179de9275a5a8417c202a0d87ec39717af0bd400f3d111cb8fa3" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true } + ] + }, + { + "id": "tools-core.live.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\",\"description\":\"patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "005ec44b618015bc42cf465ac91df5028bfe591cd0b9f0c99dfc408883584cc5" }, + "fixture": { "id": "tools-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\"}]}", "digest": "037eee2f48fad832ddfa59e9ea679878d034ad1bbb087c7511d969253e5f1f49" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "codex-core.live.tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-tool-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-live-tool-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "ebfe070989d76b0cacbaf16379ac9fb0d51d5103a2778e994c03495498484386" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "codex-core.live.custom-tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "c30efedea137e24e0e259ac5dd8a44ed5d329454e37e68a00b6de280ec61682c" }, + "fixture": { "id": "codex-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"PATCH\"}]}", "digest": "b5fc9cc273b8581f62f17b83d51e5b565f379ef3950865e687b91d9b3ee946a6" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"PATCH","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "vision-core.live.synthetic-ocr", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, + "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, + "assertions": [ + { "id": "nonce", "operator": "json_path_equals", "selector": "/client/response/json/output/0/content/0/text", "expected": "{\"nonce\":\"OCR-NONCE-42\"}", "required": true } + ] + }, + { + "id": "reasoning-core.live.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector", "reasoning_replay", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "reasoning-live-replay-vector", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true }, + { "id": "private-absent", "operator": "json_path_absent", "selector": "/client/response/json/reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.live.synthetic-tool", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "mcp-live-stub", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"ECHO\"}],\"isError\":false}}", "digest": "dd8285346f7f93b9529e2fbaabb8ffc420ca2ce0c04ca19b656ed286268fcbd4" }, + "assertions": [ + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"ECHO"}],"isError":false}, "required": true } + ] + } + ] +} diff --git a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md new file mode 100644 index 000000000..bacfc4abe --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md @@ -0,0 +1,36 @@ +# CL-03 implementation record + +## Branch + +- `feat/cl-03-live-route-probes` +- Starting SHA: `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) + +## Scope delivered + +1. Live manifest authority (`023_live_v1_manifest_authority.md`, `024_live_v1_cases.json`, runtime copy) +2. Route subject builder (`src/lab/subject/`) +3. Live sandbox (`src/lab/live/`) +4. Persistence seam (`src/lab/observe/from-live.ts`) +5. Projection applicability (`routeSubjectApplicableToRequirements`) +6. Tests (`tests/lab-live-sandbox.test.ts`, `tests/lab-live-probe.test.ts`) + +## Explicitly not started + +- CL-04 CLI/API + +## Frozen live scenarios (10) + +- `responses-core.live.basic-turn` +- `chat-core.live.basic-turn` +- `anthropic-core.live.basic-turn` +- `tools-core.live.function-round-trip` +- `tools-core.live.custom-freeform-round-trip` +- `codex-core.live.tool-turn` +- `codex-core.live.custom-tool-turn` +- `vision-core.live.synthetic-ocr` +- `reasoning-core.live.replay` +- `mcp-core.live.synthetic-tool` + +## Acceptance status + +Implementation only — not accepted. diff --git a/scripts/gen-live-digests.ts b/scripts/gen-live-digests.ts new file mode 100644 index 000000000..eeda8dcab --- /dev/null +++ b/scripts/gen-live-digests.ts @@ -0,0 +1,52 @@ +import { fixtureDigest } from "../src/lab/conformance/digest"; + +const fixtures: Record = { + "rsp-live-basic-request": + '{"model":"fixture-model","input":"PING","stream":false}', + "rsp-live-basic-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"OK"}]}]}', + "chat-live-basic-request": + '{"model":"fixture-model","input":"PING","stream":true}', + "chat-live-basic-upstream": + 'data: {"choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + "anthropic-live-basic-request": + '{"model":"fixture-model","messages":[{"role":"user","content":"PING"}],"max_tokens":32,"stream":true}', + "anthropic-live-basic-upstream": + 'data:{"type":"response.output_text.delta","delta":"OK"}\n\ndata:{"type":"response.completed","response":{"id":"resp_fixture","status":"completed","usage":{"input_tokens":1,"output_tokens":1}}}\n\n', + "tools-live-fn-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"function","name":"lookup","parameters":{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}}],"tool_choice":{"type":"function","name":"lookup"}}', + "tools-live-fn-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"function_call","call_id":"call_fixture","name":"lookup","arguments":"{\\"q\\":\\"x\\"}"}]}', + "tools-live-fn-result-request": + '{"model":"fixture-model","input":[{"type":"function_call_output","call_id":"call_fixture","output":"RESULT"}],"stream":false}', + "tools-live-fn-result-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"DONE"}]}]}', + "tools-live-custom-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"custom","name":"apply_patch","description":"patch"}],"tool_choice":{"type":"custom","name":"apply_patch"}}', + "tools-live-custom-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"custom_tool_call","call_id":"call_fixture","name":"apply_patch","input":"*** Begin Patch\\n*** End Patch"}]}', + "tools-live-custom-result-request": + '{"model":"fixture-model","input":[{"type":"custom_tool_call_output","call_id":"call_fixture","output":"PATCHED"}],"stream":false}', + "tools-live-custom-result-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"DONE"}]}]}', + "codex-live-tool-request": + '{"model":"fixture-model","input":"PING","stream":true}', + "codex-live-tool-upstream": + 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_fixture","function":{"name":"lookup","arguments":"{\\"q\\":\\"x\\"}"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n', + "codex-live-custom-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"custom","name":"apply_patch"}],"tool_choice":{"type":"custom","name":"apply_patch"}}', + "codex-live-custom-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"custom_tool_call","call_id":"call_fixture","name":"apply_patch","input":"PATCH"}]}', + "vision-live-ocr-request": + '{"model":"fixture-model","input":[{"role":"user","content":[{"type":"input_text","text":"OCR-NONCE-42"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgo=","detail":"high"}]}],"stream":false}', + "vision-live-ocr-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"{\\"nonce\\":\\"OCR-NONCE-42\\"}"}]}]}', + "reasoning-live-replay-vector": + '{"turn1":{"reasoning":{"id":"rs_fixture","text":"PLAN","signature":"sig_fixture"},"toolCall":{"callId":"call_fixture"}},"turn2":{"toolResult":{"callId":"call_fixture","output":"RESULT"}}}', + "mcp-live-stub": + '{"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"},"result":{"content":[{"type":"text","text":"ECHO"}],"isError":false}}', +}; + +for (const [id, bytes] of Object.entries(fixtures)) { + console.log(id, fixtureDigest(new TextEncoder().encode(bytes))); +} diff --git a/src/lab/conformance/fixtures/live-v1-cases.json b/src/lab/conformance/fixtures/live-v1-cases.json new file mode 100644 index 000000000..ded71e84a --- /dev/null +++ b/src/lab/conformance/fixtures/live-v1-cases.json @@ -0,0 +1,176 @@ +{ + "schemaVersion": 1, + "authority": "CL-03 live manifest authority", + "sourceCommit": "4f746d13799888ea0a8c7a111aa2ad61c2126ea0", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "live-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "auth-blocked", "match": ["auth_blocked", "authentication_blocked"], "classification": "authentication_blocked", "secondaryCode": "auth_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "quota-blocked", "match": ["quota_blocked"], "classification": "quota_blocked", "secondaryCode": "quota_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "network-blocked", "match": ["network_blocked", "network_failure"], "classification": "network_failure", "secondaryCode": "network_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "region-blocked", "match": ["region_blocked"], "classification": "region_blocked", "secondaryCode": "region_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "transient-blocked", "match": ["provider_transient"], "classification": "provider_transient", "secondaryCode": "provider_transient", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "live_route_compatibility", + "verificationRole": "required", + "executionMode": "live", + "freshness": { "maxAgeMs": 604800000 }, + "executionLimits": { + "totalTimeoutMs": 120000, + "connectTimeoutMs": 10000, + "firstByteTimeoutMs": 30000, + "inactivityTimeoutMs": 30000, + "maxRequests": 16, + "maxInputBytes": 8388608, + "maxOutputBytes": 16777216, + "maxOutputTokens": 32768, + "maxToolCalls": 32, + "maxMemoryBytes": 536870912, + "maxChildProcesses": 0, + "maxArtifacts": 16, + "perArtifactBytes": 262144, + "aggregateArtifactBytes": 1048576, + "maxArtifactBytes": 262144 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_live_v1" + }, + "failureRuleSet": "live-v1-default" + }, + "cases": [ + { + "id": "responses-core.live.basic-turn", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "rsp-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "rsp-live-basic-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]}", "digest": "8008bc78564ccf7676fba34035df94bf3f342f186ad7778f75efa71058604429" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.live.basic-turn", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "chat-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.live.basic-turn", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "anthropic-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "tools-core.live.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-fn-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"function\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"tool_choice\":{\"type\":\"function\",\"name\":\"lookup\"}}", "digest": "50b83cb763832dee3c68def08fd478c40eaf52ee2f653392e9a42935eea4a682" }, + "fixture": { "id": "tools-live-fn-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"function_call\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}]}", "digest": "18f02e77d01a179de9275a5a8417c202a0d87ec39717af0bd400f3d111cb8fa3" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true } + ] + }, + { + "id": "tools-core.live.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\",\"description\":\"patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "005ec44b618015bc42cf465ac91df5028bfe591cd0b9f0c99dfc408883584cc5" }, + "fixture": { "id": "tools-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\"}]}", "digest": "037eee2f48fad832ddfa59e9ea679878d034ad1bbb087c7511d969253e5f1f49" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "codex-core.live.tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-tool-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-live-tool-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "ebfe070989d76b0cacbaf16379ac9fb0d51d5103a2778e994c03495498484386" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "codex-core.live.custom-tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "c30efedea137e24e0e259ac5dd8a44ed5d329454e37e68a00b6de280ec61682c" }, + "fixture": { "id": "codex-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"PATCH\"}]}", "digest": "b5fc9cc273b8581f62f17b83d51e5b565f379ef3950865e687b91d9b3ee946a6" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"PATCH","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "vision-core.live.synthetic-ocr", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, + "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, + "assertions": [ + { "id": "nonce", "operator": "json_path_equals", "selector": "/client/response/json/output/0/content/0/text", "expected": "{\"nonce\":\"OCR-NONCE-42\"}", "required": true } + ] + }, + { + "id": "reasoning-core.live.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector", "reasoning_replay", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "reasoning-live-replay-vector", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true }, + { "id": "private-absent", "operator": "json_path_absent", "selector": "/client/response/json/reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.live.synthetic-tool", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "mcp-live-stub", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"ECHO\"}],\"isError\":false}}", "digest": "dd8285346f7f93b9529e2fbaabb8ffc420ca2ce0c04ca19b656ed286268fcbd4" }, + "assertions": [ + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"ECHO"}],"isError":false}, "required": true } + ] + } + ] +} diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 6bdb44fbd..872accc19 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -13,6 +13,11 @@ export type FailureClassification = | "protocol_failure" | "capability_failure" | "behavioral_failure" + | "authentication_blocked" + | "quota_blocked" + | "region_blocked" + | "network_failure" + | "provider_transient" | "inconclusive"; export interface FixtureRecord { @@ -161,6 +166,18 @@ export interface ScenarioRunResult { executionContext: ProtocolExecutionContextV1; } +/** All eight live-route compatibility suites frozen by CL-03 Live V1. */ +export const CL03_LIVE_SUITES = [ + "responses-core", + "chat-core", + "anthropic-core", + "tools-core", + "codex-core", + "vision-core", + "reasoning-core", + "mcp-core", +] as const; + /** All eight protocol-conformance suites frozen by CL-00 Protocol V1. */ export const CL01_SUITES = [ "responses-core", diff --git a/src/lab/index.ts b/src/lab/index.ts index 2515f8481..bbce80e51 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -17,3 +17,19 @@ export * from "./conformance/suite-manifest"; export * from "./ledger/artifact-refs"; export * from "./projection/verification"; export * from "./observe/from-conformance"; +export * from "./observe/from-live"; +export * from "./live/manifest"; +export * from "./live/runner"; +export * from "./live/executor"; +export * from "./live/types"; +export * from "./live/destination"; +export * from "./live/sandbox"; +export * from "./live/transport"; +export * from "./live/credential-lease"; +export * from "./live/inert-tools"; +export * from "./live/mcp-loopback"; +export * from "./live/suite-manifest"; +export * from "./subject/route-subject"; +export * from "./subject/behavior-fingerprint"; +export * from "./subject/installation-salt"; +export { CL03_LIVE_SUITES } from "./conformance/types"; diff --git a/src/lab/live/credential-lease.ts b/src/lab/live/credential-lease.ts new file mode 100644 index 000000000..9934de64e --- /dev/null +++ b/src/lab/live/credential-lease.ts @@ -0,0 +1,59 @@ +import { LAB_CREDENTIAL_LEASE, type LabCredentialLeaseV1, type LabDestinationV1 } from "./types"; + +export class LabCredentialError extends Error { + override readonly name = "LabCredentialError"; + constructor( + message: string, + readonly code: "auth_blocked" | "harness_failure", + ) { + super(message); + } +} + +export interface CreateLeaseOptions { + destination: LabDestinationV1; + transportId?: string; + budget?: number; + authOutcome?: "ok" | "blocked"; +} + +/** Create an opaque credential lease bound to destination + transport + budget. */ +export function createCredentialLease(opts: CreateLeaseOptions): LabCredentialLeaseV1 { + if (opts.authOutcome === "blocked") { + throw new LabCredentialError("credential rejected for destination", "auth_blocked"); + } + const budget = opts.budget ?? 1; + let remaining = budget; + const lease: LabCredentialLeaseV1 = { + [LAB_CREDENTIAL_LEASE]: true, + destinationFingerprint: opts.destination.fingerprint, + transportId: opts.transportId ?? "default", + get remainingRequests() { + return remaining; + }, + consume() { + if (remaining <= 0) { + throw new LabCredentialError("credential lease exhausted", "auth_blocked"); + } + remaining -= 1; + }, + }; + return Object.freeze(lease); +} + +export function isCredentialLease(value: unknown): value is LabCredentialLeaseV1 { + return typeof value === "object" && value !== null && (value as LabCredentialLeaseV1)[LAB_CREDENTIAL_LEASE] === true; +} + +export function assertLeaseScope( + lease: LabCredentialLeaseV1, + destination: LabDestinationV1, + transportId = "default", +): void { + if (lease.destinationFingerprint !== destination.fingerprint) { + throw new LabCredentialError("lease destination mismatch", "auth_blocked"); + } + if (lease.transportId !== transportId) { + throw new LabCredentialError("lease transport mismatch", "auth_blocked"); + } +} diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts new file mode 100644 index 000000000..465f79b63 --- /dev/null +++ b/src/lab/live/destination.ts @@ -0,0 +1,117 @@ +import { assessUrlDestination } from "../../lib/destination-policy"; +import { localFingerprint } from "../digest"; +import { readInstallationSalt } from "../subject/installation-salt"; +import type { DnsResolver, LabDestinationV1 } from "./types"; + +export class LabDestinationError extends Error { + override readonly name = "LabDestinationError"; + constructor( + message: string, + readonly code: string, + ) { + super(message); + } +} + +function normalizeBasePath(pathname: string): string { + if (!pathname || pathname === "/") return ""; + return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname; +} + +function destinationFingerprintValue(snapshot: { + scheme: string; + host: string; + port: number; + basePath: string; +}): Record { + return { + scheme: snapshot.scheme, + host: snapshot.host, + port: snapshot.port, + basePath: snapshot.basePath, + }; +} + +export interface CreateDestinationOptions { + baseUrl: string; + allowPrivateNetwork?: boolean; + labRunApproval?: boolean; + resolve?: DnsResolver; + configDir?: string; +} + +/** Create an immutable destination snapshot from provider baseUrl with DNS policy checks. */ +export async function createLabDestination( + opts: CreateDestinationOptions, +): Promise { + let parsed: URL; + try { + parsed = new URL(opts.baseUrl); + } catch { + throw new LabDestinationError("invalid provider baseUrl", "harness_failure"); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new LabDestinationError("unsupported URL scheme", "harness_failure"); + } + const assessment = assessUrlDestination(parsed.toString()); + if (assessment?.kind === "metadata" || assessment?.kind === "link-local" || assessment?.kind === "unspecified") { + throw new LabDestinationError(`blocked destination: ${assessment.detail}`, "harness_failure"); + } + const privateNetwork = assessment?.kind === "private" || assessment?.kind === "loopback" || assessment?.kind === "localhost"; + if (privateNetwork && (!opts.allowPrivateNetwork || !opts.labRunApproval)) { + throw new LabDestinationError("private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); + } + + const resolve = opts.resolve ?? defaultResolver; + const addresses = await resolve(parsed.hostname); + if (addresses.length === 0) { + throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); + } + + const port = parsed.port + ? Number(parsed.port) + : parsed.protocol === "https:" ? 443 : 80; + const snapshot = { + scheme: parsed.protocol === "https:" ? "https" as const : "http" as const, + host: parsed.hostname.toLowerCase(), + port, + basePath: normalizeBasePath(parsed.pathname), + sniHost: parsed.hostname.toLowerCase(), + addresses: Object.freeze(addresses.map((a) => Object.freeze({ ...a }))), + privateNetwork, + }; + const salt = readInstallationSalt(opts.configDir); + const fingerprint = localFingerprint("endpoint", destinationFingerprintValue(snapshot), salt); + return Object.freeze({ ...snapshot, fingerprint }); +} + +/** Fail closed when a re-resolution does not match the immutable snapshot. */ +export function assertDestinationAddressSet( + destination: LabDestinationV1, + addresses: Array<{ address: string; family: 4 | 6 }>, +): void { + const a = [...destination.addresses].map((x) => `${x.family}:${x.address}`).sort().join(","); + const b = [...addresses].map((x) => `${x.family}:${x.address}`).sort().join(","); + if (a !== b) { + throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); + } +} + +export function assertHostSniMatch(destination: LabDestinationV1, host: string, sni?: string): void { + const normalized = host.toLowerCase(); + if (normalized !== destination.host) { + throw new LabDestinationError("host mismatch", "host_sni_mismatch"); + } + if (sni !== undefined && sni.toLowerCase() !== destination.sniHost) { + throw new LabDestinationError("SNI mismatch", "host_sni_mismatch"); + } +} + +async function defaultResolver(hostname: string): Promise> { + const { lookup } = await import("node:dns/promises"); + const results = await lookup(hostname, { all: true, verbatim: true }); + return results.map((r) => ({ + address: r.address, + family: r.family === 6 ? 6 as const : 4 as const, + })); +} diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts new file mode 100644 index 000000000..2385d9b2e --- /dev/null +++ b/src/lab/live/executor.ts @@ -0,0 +1,276 @@ +import { evaluateAssertions } from "../conformance/assertion"; +import { executeScenario } from "../conformance/executor"; +import type { CaseRecord, FailureClassification, FailureRule } from "../conformance/types"; +import type { RouteSubjectV1 } from "../events/types"; +import { createCredentialLease, LabCredentialError } from "./credential-lease"; +import { createLabDestination } from "./destination"; +import { expandLiveScenario } from "./manifest"; +import { buildRouteSubjectV1, freezeRouteSubject } from "../subject/route-subject"; +import { + createSandboxResourceState, + enforceSandboxLimits, + prepareLiveSandbox, +} from "./sandbox"; +import { classifyTransportError, createMockTransport, TransportError } from "./transport"; +import type { + DnsResolver, + LabRouteContext, + LabTransport, + LiveRunConfig, + LiveScenarioRunResult, +} from "./types"; + +export interface LiveExecutorOptions { + transport?: LabTransport; + resolve?: DnsResolver; + configDir?: string; + authOutcome?: "ok" | "blocked"; + env?: NodeJS.ProcessEnv; +} + +function liveLimitsFromAuthority(caseRecord: CaseRecord, authorityLimits: Record): LiveRunConfig { + return { + totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, + connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, + firstByteTimeoutMs: authorityLimits.firstByteTimeoutMs ?? 30_000, + inactivityTimeoutMs: authorityLimits.inactivityTimeoutMs ?? 30_000, + maxRequests: authorityLimits.maxRequests ?? 16, + maxInputBytes: authorityLimits.maxInputBytes ?? 8 * 1024 * 1024, + maxOutputBytes: authorityLimits.maxOutputBytes ?? 16 * 1024 * 1024, + maxOutputTokens: authorityLimits.maxOutputTokens ?? 32_768, + maxToolCalls: authorityLimits.maxToolCalls ?? 32, + maxMemoryBytes: authorityLimits.maxMemoryBytes ?? 512 * 1024 * 1024, + maxChildProcesses: authorityLimits.maxChildProcesses ?? 0, + maxArtifacts: authorityLimits.maxArtifacts ?? 16, + perArtifactBytes: authorityLimits.perArtifactBytes ?? 256 * 1024, + aggregateArtifactBytes: authorityLimits.aggregateArtifactBytes ?? 1024 * 1024, + }; +} + +function checkRoutePreconditions(routeContext: LabRouteContext, caseRecord: CaseRecord): string | null { + for (const precondition of caseRecord.requirements.routePreconditions) { + if (precondition === "lab_run_approval" && routeContext.labRunApproval !== true) { + return "route_precondition_unmet:lab_run_approval"; + } + if (precondition === "allow_private_network" && routeContext.allowPrivateNetwork !== true) { + return "route_precondition_unmet:allow_private_network"; + } + } + for (const claim of caseRecord.requirements.requiredClaims) { + if (!(routeContext.requiredClaims ?? []).includes(claim)) { + return `required_claim_missing:${claim}`; + } + } + return null; +} + +function transportFromFixtures(caseRecord: CaseRecord): LabTransport { + const entries = []; + if (caseRecord.initiatingRequest && caseRecord.fixture.role === "upstream_response") { + entries.push({ + status: 200, + headers: { "content-type": caseRecord.fixture.mediaType }, + body: caseRecord.fixture.bytesUtf8, + }); + } else if (caseRecord.fixture.role === "upstream_response") { + entries.push({ + status: 200, + headers: { "content-type": caseRecord.fixture.mediaType }, + body: caseRecord.fixture.bytesUtf8, + }); + } else if (caseRecord.fixture.role === "adapter_vector" || caseRecord.fixture.role === "client_request") { + entries.push({ status: 200, body: caseRecord.fixture.bytesUtf8 }); + } else if (caseRecord.fixture.role === "synthetic_tool") { + entries.push({ status: 200, body: "{}" }); + } + return createMockTransport({ entries }); +} + +function classifyWithFailureRules( + rules: FailureRule[], + signal: string, + defaultClass: FailureClassification, + defaultCode: string, +): { classification: FailureClassification; secondaryCode: string } { + for (const rule of rules) { + if (rule.match.includes(signal) || rule.match.includes("no_prior_rule")) { + if (rule.match.includes("no_prior_rule") && signal !== "no_prior_rule") continue; + if (!rule.match.includes(signal) && !rule.match.includes("no_prior_rule")) continue; + return { + classification: rule.classification, + secondaryCode: rule.secondaryCode ?? defaultCode, + }; + } + } + return { classification: defaultClass, secondaryCode: defaultCode }; +} + +async function executeLiveObservation( + caseRecord: CaseRecord, + transport: LabTransport, + limits: LiveRunConfig, +): Promise<{ observation: Awaited> }> { + const state = createSandboxResourceState(); + if (caseRecord.fixture.role !== "adapter_vector" && caseRecord.fixture.role !== "synthetic_tool") { + const response = await transport.request({ + method: "POST", + path: "/chat/completions", + body: caseRecord.initiatingRequest?.bytesUtf8 ?? caseRecord.fixture.bytesUtf8, + }); + enforceSandboxLimits(state, limits, { + requests: 1, + inputBytes: (caseRecord.initiatingRequest?.bytesUtf8 ?? caseRecord.fixture.bytesUtf8).length, + outputBytes: response.body.length, + }); + } + const observation = await executeScenario(caseRecord); + return { observation }; +} + +/** Run one live scenario against a route context with injectable transport. */ +export async function runLiveScenario( + caseRecord: CaseRecord, + routeContext: LabRouteContext, + opts: LiveExecutorOptions = {}, +): Promise { + const diagnostics: string[] = []; + const startedAt = Date.now(); + const complete = ( + partial: Omit, + ): LiveScenarioRunResult => ({ + ...partial, + startedAt, + completedAt: Math.max(startedAt, Date.now()), + }); + + try { + prepareLiveSandbox(opts.env); + const destination = await createLabDestination({ + baseUrl: routeContext.baseUrl, + allowPrivateNetwork: routeContext.allowPrivateNetwork, + labRunApproval: routeContext.labRunApproval, + resolve: opts.resolve, + configDir: opts.configDir, + }); + const routeSubject = freezeRouteSubject( + buildRouteSubjectV1(routeContext, destination, opts.configDir), + ); + const preconditionFailure = checkRoutePreconditions(routeContext, caseRecord); + if (preconditionFailure) { + return complete({ + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: false, + classification: "inconclusive", + secondaryCode: preconditionFailure, + assertionResults: [], + diagnostics: [preconditionFailure], + routeSubject, + }); + } + + const lease = createCredentialLease({ + destination, + authOutcome: opts.authOutcome ?? "ok", + }); + void lease; + + const authority = await import("./manifest").then((m) => m.loadLiveCaseAuthority()); + const expanded = expandLiveScenario(caseRecord, authority); + const limits = liveLimitsFromAuthority(caseRecord, expanded.executionLimits as Record); + const transport = opts.transport ?? transportFromFixtures(caseRecord); + + const { observation } = await executeLiveObservation(caseRecord, transport, limits); + const assertionResults = evaluateAssertions(caseRecord.assertions, observation); + const requiredFailures = assertionResults.filter((r) => r.required && !r.passed); + + if (caseRecord.expectedFailure) { + const listed = caseRecord.expectedFailure.assertionIds; + const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed === true); + const expectedFailureMatched = controlPassed && requiredFailures.length === 0; + return complete({ + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: expectedFailureMatched, + classification: expectedFailureMatched + ? caseRecord.expectedFailure.expectedClass as FailureClassification + : "protocol_failure", + secondaryCode: expectedFailureMatched + ? caseRecord.expectedFailure.expectedCode + : "deterministic_assertion", + assertionResults, + diagnostics, + routeSubject, + }); + } + + const passed = requiredFailures.length === 0; + const failureRules = expanded.failureRules as FailureRule[]; + const signal = passed ? "pass" : "required_assertion_failed"; + const classified = passed + ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass" } + : classifyWithFailureRules(failureRules, signal, "protocol_failure", "deterministic_assertion"); + + return complete({ + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed, + classification: passed ? "inconclusive" : classified.classification, + secondaryCode: passed ? undefined : classified.secondaryCode, + assertionResults, + diagnostics, + routeSubject, + }); + } catch (error) { + diagnostics.push(String(error)); + if (error instanceof LabCredentialError || (error instanceof Error && error.name === "LabCredentialError")) { + const credError = error as LabCredentialError; + return complete({ + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: false, + classification: "authentication_blocked", + secondaryCode: credError.code, + assertionResults: [], + diagnostics, + routeSubject: fallbackRouteSubject(routeContext), + transportError: "auth_blocked", + }); + } + const transportCode = error instanceof TransportError + || (error instanceof Error && error.name === "TransportError") + ? (error as TransportError).code + : undefined; + const classified = classifyTransportError(error); + return complete({ + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: false, + classification: classified.classification, + secondaryCode: classified.secondaryCode, + assertionResults: [], + diagnostics, + routeSubject: fallbackRouteSubject(routeContext), + transportError: transportCode, + }); + } +} + +function fallbackRouteSubject(routeContext: LabRouteContext): RouteSubjectV1 { + return { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: routeContext.providerId, + providerInstanceFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", + clientModelId: routeContext.clientModelId, + upstreamModelId: routeContext.upstreamModelId, + effectiveAdapter: routeContext.effectiveAdapter, + inboundProtocol: routeContext.inboundProtocol, + upstreamProtocol: routeContext.upstreamProtocol, + surface: routeContext.surface, + opencodexCompatibilityVersion: "live-v1", + behaviorFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", + endpointFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", + dependencies: [], + }; +} diff --git a/src/lab/live/inert-tools.ts b/src/lab/live/inert-tools.ts new file mode 100644 index 000000000..8f6b33b07 --- /dev/null +++ b/src/lab/live/inert-tools.ts @@ -0,0 +1,56 @@ +export interface InertToolDefinition { + name: string; + kind: "function" | "custom"; + description?: string; + parameters?: Record; +} + +export interface InertToolResult { + output: string; +} + +const LAB_TOOLS: Record = { + lookup: { + name: "lookup", + kind: "function", + description: "Lab inert lookup tool", + parameters: { + type: "object", + properties: { q: { type: "string" } }, + required: ["q"], + }, + staticResult: { output: "RESULT" }, + }, + apply_patch: { + name: "apply_patch", + kind: "custom", + description: "Lab inert patch tool", + staticResult: { output: "PATCHED" }, + }, +}; + +export function listInertToolDefinitions(): InertToolDefinition[] { + return Object.values(LAB_TOOLS).map(({ staticResult: _r, ...tool }) => tool); +} + +export function validateInertToolArgs(name: string, args: unknown): void { + const tool = LAB_TOOLS[name]; + if (!tool) throw new Error(`unknown inert tool: ${name}`); + if (tool.kind === "function") { + if (!args || typeof args !== "object" || Array.isArray(args)) { + throw new Error("invalid function tool arguments"); + } + const record = args as Record; + if (typeof record.q !== "string") { + throw new Error("lookup requires string q"); + } + } +} + +/** Execute an inert lab tool — validate args and return static results only. */ +export function executeInertTool(name: string, args: unknown): InertToolResult { + validateInertToolArgs(name, args); + const tool = LAB_TOOLS[name]; + if (!tool) throw new Error(`unknown inert tool: ${name}`); + return { ...tool.staticResult }; +} diff --git a/src/lab/live/manifest.ts b/src/lab/live/manifest.ts new file mode 100644 index 000000000..eda89fb55 --- /dev/null +++ b/src/lab/live/manifest.ts @@ -0,0 +1,123 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { fixtureDigest } from "../conformance/digest"; +import type { + CaseAuthority, + CaseRecord, + FailureClassification, + FailureRule, +} from "../conformance/types"; +import { CL03_LIVE_SUITES, SYNTHETIC_MARKER } from "../conformance/types"; + +const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +const AUTHORITY_FILE = "024_live_v1_cases.json"; +const FIXTURES_DIR = join(MODULE_DIR, "..", "conformance", "fixtures"); + +export function loadLiveCaseAuthority(): CaseAuthority { + const path = join(FIXTURES_DIR, "live-v1-cases.json"); + const raw = JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; + validateLiveAuthority(raw); + return raw; +} + +export function discoverLiveScenarios( + authority: CaseAuthority, + suites: readonly string[] = CL03_LIVE_SUITES, +): CaseRecord[] { + return authority.cases.filter((c) => suites.includes(c.suite)); +} + +export function expandLiveScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { + const defaults = authority.manifestDefaults; + const fixtures = caseRecord.initiatingRequest + ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] + : [fixtureRef(caseRecord.fixture, authority)]; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.id, + version: defaults.version, + suite: { + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + }, + evidenceLayer: defaults.evidenceLayer, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, + requirements: caseRecord.requirements, + fixtures, + executionLimits: defaults.executionLimits, + executionMode: defaults.executionMode, + assertions: caseRecord.assertions, + ...(caseRecord.expectedFailure ? { expectedFailure: caseRecord.expectedFailure } : {}), + failureRules: expandLiveFailureRules(caseRecord, authority), + artifactPolicy: defaults.artifactPolicy, + freshness: defaults.freshness, + }; +} + +function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): Record { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + return { + id: fixture.id, + role: fixture.role, + mediaType: fixture.mediaType, + digest: fixture.digest, + byteLength: bytes.byteLength, + syntheticMarker: SYNTHETIC_MARKER, + provenance: { + kind: "lab_authored", + authority: AUTHORITY_FILE, + sourceCommit: authority.sourceCommit, + }, + }; +} + +function expandLiveFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { + const setName = authority.manifestDefaults.failureRuleSet; + const ruleSet = authority.failureRuleSets[setName]; + if (!Array.isArray(ruleSet)) { + throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); + } + const base = [...ruleSet]; + if (!caseRecord.expectedFailure) return base; + const template = authority.expectedFailureRuleTemplate; + const controlRule: FailureRule = { + id: template.id, + match: [...template.match], + classification: caseRecord.expectedFailure.expectedClass as FailureClassification, + secondaryCode: caseRecord.expectedFailure.expectedCode, + verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", + retry: template.retry, + expected: template.expected, + }; + const idx = base.findIndex((r) => r.id === "required-assertion"); + if (idx >= 0) base.splice(idx, 0, controlRule); + else base.push(controlRule); + return base; +} + +function validateLiveAuthority(authority: CaseAuthority): void { + if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); + if (!authority.sourceCommit || typeof authority.sourceCommit !== "string") { + throw new Error("missing sourceCommit"); + } + if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); + if (authority.manifestDefaults.evidenceLayer !== "live_route_compatibility") { + throw new Error("invalid evidenceLayer for live authority"); + } + for (const caseRecord of authority.cases) { + const bytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + if (fixtureDigest(bytes) !== caseRecord.fixture.digest) { + throw new Error(`${caseRecord.id}: fixture digest mismatch`); + } + if (caseRecord.initiatingRequest) { + const initBytes = new TextEncoder().encode(caseRecord.initiatingRequest.bytesUtf8); + if (fixtureDigest(initBytes) !== caseRecord.initiatingRequest.digest) { + throw new Error(`${caseRecord.id}: initiatingRequest digest mismatch`); + } + } + expandLiveScenario(caseRecord, authority); + } +} diff --git a/src/lab/live/mcp-loopback.ts b/src/lab/live/mcp-loopback.ts new file mode 100644 index 000000000..3d08c61ca --- /dev/null +++ b/src/lab/live/mcp-loopback.ts @@ -0,0 +1,57 @@ +export interface McpStubTool { + namespace: string; + name: string; + inputSchema?: Record; +} + +export interface McpStubCall { + namespace: string; + name: string; + arguments: Record; +} + +export interface McpStubResult { + content: Array<{ type: "text"; text: string }>; +} + +const tools = new Map(); +const invocations: McpStubCall[] = []; + +function toolKey(namespace: string, name: string): string { + return `${namespace}::${name}`; +} + +/** Register a lab-owned MCP stub tool (in-memory only). */ +export function registerMcpStubTool(tool: McpStubTool): void { + tools.set(toolKey(tool.namespace, tool.name), { ...tool }); +} + +export function clearMcpStub(): void { + tools.clear(); + invocations.length = 0; +} + +export function listMcpStubTools(): McpStubTool[] { + return [...tools.values()]; +} + +/** Invoke the in-memory MCP stub with fixed pure results. */ +export function invokeMcpStub( + namespace: string, + name: string, + args: Record, + result: McpStubResult, +): McpStubResult { + const key = toolKey(namespace, name); + if (!tools.has(key)) throw new Error(`MCP stub tool not registered: ${namespace}/${name}`); + invocations.push({ namespace, name, arguments: { ...args } }); + return result; +} + +export function mcpStubInvocations(): readonly McpStubCall[] { + return invocations; +} + +export function wireName(namespace: string, name: string): string { + return `mcp__${namespace}__${name}`; +} diff --git a/src/lab/live/runner.ts b/src/lab/live/runner.ts new file mode 100644 index 000000000..578c962fe --- /dev/null +++ b/src/lab/live/runner.ts @@ -0,0 +1,32 @@ +import { discoverLiveScenarios, loadLiveCaseAuthority } from "./manifest"; +import { runLiveScenario, type LiveExecutorOptions } from "./executor"; +import type { LabRouteContext, LiveScenarioRunResult } from "./types"; +import { CL03_LIVE_SUITES } from "../conformance/types"; + +export interface LiveRunSummary { + total: number; + passed: number; + failed: number; + results: LiveScenarioRunResult[]; +} + +export async function runLiveSuite( + routeContext: LabRouteContext, + suites: readonly string[] = CL03_LIVE_SUITES, + opts: LiveExecutorOptions = {}, +): Promise { + const authority = loadLiveCaseAuthority(); + const scenarios = discoverLiveScenarios(authority, suites); + if (scenarios.length === 0) throw new Error("harness_failure: no CL-03 live scenarios discovered"); + const results: LiveScenarioRunResult[] = []; + for (const scenario of scenarios) { + results.push(await runLiveScenario(scenario, routeContext, opts)); + } + const passed = results.filter((r) => r.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export function listLiveScenarioIds(suites: readonly string[] = CL03_LIVE_SUITES): string[] { + const authority = loadLiveCaseAuthority(); + return discoverLiveScenarios(authority, suites).map((s) => s.id); +} diff --git a/src/lab/live/sandbox.ts b/src/lab/live/sandbox.ts new file mode 100644 index 000000000..5116f08d5 --- /dev/null +++ b/src/lab/live/sandbox.ts @@ -0,0 +1,108 @@ +import type { LiveRunConfig } from "./types"; + +const PROXY_ENV_VARS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", +] as const; + +const ALLOWED_ENV = { + TZ: "UTC", + NO_COLOR: "1", +} as const; + +export class LabSandboxError extends Error { + override readonly name = "LabSandboxError"; + constructor( + message: string, + readonly code: string, + ) { + super(message); + } +} + +/** Reject inherited proxy environment variables for Lab live runs. */ +export function rejectProxyEnvironment(env: NodeJS.ProcessEnv = process.env): void { + for (const name of PROXY_ENV_VARS) { + const value = env[name]; + if (value !== undefined && value !== "") { + throw new LabSandboxError(`proxy environment variable ${name} is forbidden`, "harness_failure"); + } + } +} + +/** Build the constructed non-inherited environment view for live runs. */ +export function labSandboxEnvironment(): Readonly> { + return Object.freeze({ ...ALLOWED_ENV }); +} + +export interface SandboxResourceState { + requests: number; + inputBytes: number; + outputBytes: number; + toolCalls: number; + artifacts: number; + artifactBytes: number; + childProcesses: number; +} + +export function createSandboxResourceState(): SandboxResourceState { + return { + requests: 0, + inputBytes: 0, + outputBytes: 0, + toolCalls: 0, + artifacts: 0, + artifactBytes: 0, + childProcesses: 0, + }; +} + +export function enforceSandboxLimits( + state: SandboxResourceState, + limits: LiveRunConfig, + delta: Partial = {}, +): void { + const next = { + requests: state.requests + (delta.requests ?? 0), + inputBytes: state.inputBytes + (delta.inputBytes ?? 0), + outputBytes: state.outputBytes + (delta.outputBytes ?? 0), + toolCalls: state.toolCalls + (delta.toolCalls ?? 0), + artifacts: state.artifacts + (delta.artifacts ?? 0), + artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), + childProcesses: state.childProcesses + (delta.childProcesses ?? 0), + }; + if (next.childProcesses > limits.maxChildProcesses) { + throw new LabSandboxError("child process limit exceeded", "harness_failure"); + } + if (next.requests > limits.maxRequests) { + throw new LabSandboxError("request limit exceeded", "request_limit"); + } + if (next.inputBytes > limits.maxInputBytes) { + throw new LabSandboxError("input byte limit exceeded", "request_limit"); + } + if (next.outputBytes > limits.maxOutputBytes) { + throw new LabSandboxError("output byte limit exceeded", "request_limit"); + } + if (next.toolCalls > limits.maxToolCalls) { + throw new LabSandboxError("tool call limit exceeded", "request_limit"); + } + if (next.artifacts > limits.maxArtifacts) { + throw new LabSandboxError("artifact limit exceeded", "request_limit"); + } + if (next.artifactBytes > limits.aggregateArtifactBytes) { + throw new LabSandboxError("aggregate artifact limit exceeded", "request_limit"); + } + Object.assign(state, next); +} + +/** Prepare sandbox: reject proxy env and return allowed environment. */ +export function prepareLiveSandbox(env: NodeJS.ProcessEnv = process.env): Readonly> { + rejectProxyEnvironment(env); + return labSandboxEnvironment(); +} diff --git a/src/lab/live/suite-manifest.ts b/src/lab/live/suite-manifest.ts new file mode 100644 index 000000000..efa88f53f --- /dev/null +++ b/src/lab/live/suite-manifest.ts @@ -0,0 +1,67 @@ +import { scenarioManifestDigest, suiteManifestDigest } from "../digest"; +import type { CaseAuthority, CaseRecord, VerificationRole } from "../conformance/types"; +import { expandLiveScenario } from "./manifest"; + +export interface LiveSuiteScenarioRefV1 { + id: string; + version: string; + role: VerificationRole; + manifestDigest: string; +} + +export interface LiveSuiteManifestV1 { + schemaVersion: 1; + id: string; + version: string; + evidenceLayer: string; + capability: string; + assertionDslVersion: string; + evidenceSchemaVersion: string; + freshness: { maxAgeMs: number | null }; + contradictionRule: string; + scenarios: LiveSuiteScenarioRefV1[]; + verificationRule: string; +} + +export function expandLiveSuiteManifest( + suiteId: string, + authority: CaseAuthority, +): LiveSuiteManifestV1 { + const cases = authority.cases.filter((c) => c.suite === suiteId); + if (cases.length === 0) throw new Error(`unknown live suite ${suiteId}`); + const defaults = authority.manifestDefaults; + const capability = cases[0]!.capability; + const scenarios: LiveSuiteScenarioRefV1[] = cases + .map((caseRecord) => ({ + id: caseRecord.id, + version: String(defaults.version), + role: caseRecord.verificationRole ?? defaults.verificationRole, + manifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), + })) + .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + return { + schemaVersion: 1, + id: suiteId, + version: String(defaults.suiteVersion), + evidenceLayer: defaults.evidenceLayer, + capability, + assertionDslVersion: authority.assertionDslVersion, + evidenceSchemaVersion: authority.evidenceSchemaVersion, + freshness: defaults.freshness ?? { maxAgeMs: null }, + contradictionRule: "newest-required-observation-v1", + scenarios, + verificationRule: "all-applicable-required-pass-v1", + }; +} + +export function liveSuiteManifestObjectForCase( + caseRecord: CaseRecord, + authority: CaseAuthority, +): Record { + return expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record; +} + +export function liveSuiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string { + return suiteManifestDigest(expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record); +} diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts new file mode 100644 index 000000000..4d5528d18 --- /dev/null +++ b/src/lab/live/transport.ts @@ -0,0 +1,144 @@ +import { providerRedirectError } from "../../lib/provider-outbound"; +import { assertLeaseScope } from "./credential-lease"; +import type { + LabCredentialLeaseV1, + LabDestinationV1, + LabTransport, + TransportErrorCode, + TransportRequest, + TransportResponse, +} from "./types"; + +export interface MockTransportEntry { + matchPath?: string | RegExp; + status: number; + headers?: Record; + body: string; + error?: TransportErrorCode; +} + +export interface MockTransportOptions { + entries?: MockTransportEntry[]; + onRequest?: (req: TransportRequest) => void; +} + +/** Injectable mock transport for tests — returns fixture upstream responses. */ +export function createMockTransport(opts: MockTransportOptions = {}): LabTransport { + let callIndex = 0; + return { + async request(req: TransportRequest): Promise { + opts.onRequest?.(req); + const entry = opts.entries?.[callIndex] ?? opts.entries?.[opts.entries.length - 1]; + callIndex += 1; + if (!entry) { + throw transportError("harness_failure", "no mock transport entry"); + } + if (entry.error) throw transportError(entry.error, entry.error); + if (entry.matchPath) { + const matches = typeof entry.matchPath === "string" + ? req.path.includes(entry.matchPath) + : entry.matchPath.test(req.path); + if (!matches) throw transportError("harness_failure", "mock path mismatch"); + } + if (entry.status >= 300 && entry.status < 400) { + throw transportError("redirect_blocked", "redirect response"); + } + return { + status: entry.status, + headers: { ...(entry.headers ?? {}), "content-type": entry.headers?.["content-type"] ?? "application/json" }, + body: entry.body, + }; + }, + }; +} + +export interface PinnedTransportOptions { + destination: LabDestinationV1; + lease: LabCredentialLeaseV1; + fetch?: typeof globalThis.fetch; + transportId?: string; +} + +/** Production path: provider-outbound with pinned destination; lease consumed per request. */ +export function createPinnedTransport(opts: PinnedTransportOptions): LabTransport { + const transportId = opts.transportId ?? "default"; + return { + async request(req: TransportRequest): Promise { + assertLeaseScope(opts.lease, opts.destination, transportId); + opts.lease.consume(); + const url = `${opts.destination.scheme}://${opts.destination.host}:${opts.destination.port}${opts.destination.basePath}${req.path}`; + const fetchFn = opts.fetch ?? globalThis.fetch; + const response = await fetchFn(url, { + method: req.method, + headers: req.headers, + body: req.body, + redirect: "manual", + }); + const redirectError = await providerRedirectError(response, url); + if (redirectError) throw transportError("redirect_blocked", redirectError); + if (response.status === 401 || response.status === 403) { + throw transportError("auth_blocked", `HTTP ${response.status}`); + } + if (response.status === 429) { + throw transportError("quota_blocked", `HTTP ${response.status}`); + } + if (response.status >= 500) { + throw transportError("provider_transient", `HTTP ${response.status}`); + } + const body = await response.text(); + const headers: Record = {}; + response.headers.forEach((value, key) => { + headers[key.toLowerCase()] = value; + }); + return { status: response.status, headers, body }; + }, + }; +} + +export class TransportError extends Error { + override readonly name = "TransportError"; + constructor( + readonly code: TransportErrorCode, + message: string, + ) { + super(message); + } +} + +function transportError(code: TransportErrorCode, message: string): TransportError { + return new TransportError(code, message); +} + +export function classifyTransportError(error: unknown): { classification: string; secondaryCode: string } { + if (error instanceof TransportError) { + switch (error.code) { + case "auth_blocked": + return { classification: "authentication_blocked", secondaryCode: "auth_blocked" }; + case "quota_blocked": + return { classification: "quota_blocked", secondaryCode: "quota_blocked" }; + case "network_blocked": + return { classification: "network_failure", secondaryCode: "network_blocked" }; + case "region_blocked": + return { classification: "region_blocked", secondaryCode: "region_blocked" }; + case "provider_transient": + return { classification: "provider_transient", secondaryCode: "provider_transient" }; + case "connect_timeout": + case "first_byte_timeout": + case "inactivity_timeout": + case "total_timeout": + return { classification: "timeout", secondaryCode: error.code }; + case "request_limit": + return { classification: "budget_exhausted", secondaryCode: "scenario_resource_limit" }; + case "redirect_blocked": + case "destination_mismatch": + case "host_sni_mismatch": + case "harness_failure": + return { classification: "harness_failure", secondaryCode: error.code }; + default: { + const _never: never = error.code; + return { classification: "harness_failure", secondaryCode: String(_never) }; + } + } + } + return { classification: "harness_failure", secondaryCode: "execution_error" }; +} diff --git a/src/lab/live/types.ts b/src/lab/live/types.ts new file mode 100644 index 000000000..16724ded5 --- /dev/null +++ b/src/lab/live/types.ts @@ -0,0 +1,103 @@ +import type { RouteDependencyV1 } from "../events/types"; + +export interface LabDestinationV1 { + readonly scheme: "http" | "https"; + readonly host: string; + readonly port: number; + readonly basePath: string; + readonly sniHost: string; + readonly addresses: ReadonlyArray<{ readonly address: string; readonly family: 4 | 6 }>; + readonly privateNetwork: boolean; + readonly fingerprint: string; +} + +export interface LabRouteContext { + providerId: string; + clientModelId: string; + upstreamModelId: string; + effectiveAdapter: string; + inboundProtocol: string; + upstreamProtocol: string; + surface: string; + baseUrl: string; + allowPrivateNetwork?: boolean; + labRunApproval?: boolean; + requiredClaims?: string[]; + dependencies?: RouteDependencyV1[]; +} + +export interface LiveRunConfig { + totalTimeoutMs: number; + connectTimeoutMs: number; + firstByteTimeoutMs: number; + inactivityTimeoutMs: number; + maxRequests: number; + maxInputBytes: number; + maxOutputBytes: number; + maxOutputTokens: number; + maxToolCalls: number; + maxMemoryBytes: number; + maxChildProcesses: number; + maxArtifacts: number; + perArtifactBytes: number; + aggregateArtifactBytes: number; +} + +export type DnsResolver = (hostname: string) => Promise>; + +export type TransportErrorCode = + | "auth_blocked" + | "quota_blocked" + | "network_blocked" + | "region_blocked" + | "provider_transient" + | "redirect_blocked" + | "destination_mismatch" + | "host_sni_mismatch" + | "connect_timeout" + | "first_byte_timeout" + | "inactivity_timeout" + | "total_timeout" + | "request_limit" + | "harness_failure"; + +export interface TransportRequest { + method: "GET" | "POST"; + path: string; + headers?: Record; + body?: string; +} + +export interface TransportResponse { + status: number; + headers: Record; + body: string; +} + +export interface LabTransport { + request(req: TransportRequest): Promise; +} + +export const LAB_CREDENTIAL_LEASE = Symbol("LabCredentialLeaseV1"); + +export interface LabCredentialLeaseV1 { + readonly [LAB_CREDENTIAL_LEASE]: true; + readonly destinationFingerprint: string; + readonly transportId: string; + readonly remainingRequests: number; + consume(): void; +} + +export interface LiveScenarioRunResult { + scenarioId: string; + suite: string; + startedAt: number; + completedAt: number; + passed: boolean; + classification: string; + secondaryCode?: string; + assertionResults: import("../conformance/types").AssertionResult[]; + diagnostics: string[]; + routeSubject: import("../events/types").RouteSubjectV1; + transportError?: TransportErrorCode; +} diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 606b95b86..048e92a73 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -139,6 +139,11 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { switch (result.classification) { case "timeout": case "budget_exhausted": + case "authentication_blocked": + case "quota_blocked": + case "region_blocked": + case "network_failure": + case "provider_transient": return "blocked"; case "inconclusive": case "harness_failure": diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts new file mode 100644 index 000000000..5a9215770 --- /dev/null +++ b/src/lab/observe/from-live.ts @@ -0,0 +1,238 @@ +/** + * Persistence seam: transform CL-03 live probe results into valid observation events. + */ +import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PRODUCER_VERSION, + OBSERVATION_LIMIT_NAMES, + type ObservationOutcome, +} from "../constants"; +import { fixtureDigest, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; +import type { ObservationEvent } from "../events/types"; +import { assignEventId } from "../events/validate"; +import { appendLabEvent } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import type { CaseAuthority, CaseRecord } from "../conformance/types"; +import { expandLiveScenario } from "../live/manifest"; +import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; +import type { LiveScenarioRunResult } from "../live/types"; + +export interface PersistLiveOptions { + configDir?: string; + recordedAt?: number; + startedAt?: number; + completedAt?: number; + producerVersion?: string; + artifactStore?: ArtifactStore; +} + +export interface PersistedLiveObservation { + event: ObservationEvent; + ledgerPath: string; +} + +function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcome { + if (result.passed) return "pass"; + switch (result.classification) { + case "timeout": + case "budget_exhausted": + case "authentication_blocked": + case "quota_blocked": + case "region_blocked": + case "network_failure": + case "provider_transient": + return "blocked"; + case "inconclusive": + case "harness_failure": + return "inconclusive"; + case "protocol_failure": + case "capability_failure": + case "behavioral_failure": + return "fail"; + default: + return "inconclusive"; + } +} + +function requireExecutionTimes( + result: LiveScenarioRunResult, + opts: PersistLiveOptions, +): { startedAt: number; completedAt: number } { + const startedAt = opts.startedAt ?? result.startedAt; + const completedAt = opts.completedAt ?? result.completedAt; + if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt)) { + throw new Error("real startedAt/completedAt are required for persisted live evidence"); + } + if (startedAt < 0 || completedAt < startedAt) { + throw new Error("invalid persisted live execution timestamps"); + } + return { startedAt, completedAt }; +} + +export function observationFromLiveResult( + result: LiveScenarioRunResult, + caseRecord: CaseRecord, + authority: CaseAuthority, + opts: PersistLiveOptions = {}, +): { event: ObservationEvent; artifacts: ReturnType[] } { + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const { startedAt, completedAt } = requireExecutionTimes(result, opts); + const recordedAt = opts.recordedAt ?? completedAt; + + const expandedScenario = expandLiveScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expandedScenario); + const suiteExpanded = liveSuiteManifestObjectForCase(caseRecord, authority); + const suiteDigest = suiteManifestDigest(suiteExpanded); + + const fixtureDigests: string[] = []; + const artifacts: ReturnType[] = []; + + const putFixture = (fixture: CaseRecord["fixture"]) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + const digest = fixtureDigest(bytes); + fixtureDigests.push(digest); + artifacts.push( + store.put({ + artifactClass: "fixture", + payload: bytes, + expectedDigest: digest, + mediaType: fixture.mediaType, + }), + ); + }; + putFixture(caseRecord.fixture); + if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); + + artifacts.push( + store.put({ + artifactClass: "scenario_manifest", + payload: expandedScenario, + expectedDigest: scenarioDigest, + }), + ); + artifacts.push( + store.put({ + artifactClass: "suite_manifest", + payload: suiteExpanded, + expectedDigest: suiteDigest, + }), + ); + + const assertionReport = store.put({ + artifactClass: "assertion_report", + payload: { + scenarioId: result.scenarioId, + passed: result.passed, + classification: result.classification, + assertions: result.assertionResults.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: a.passed, + observedSummary: a.observedSummary, + reason: a.reason, + })), + }, + }); + artifacts.push(assertionReport); + + const subject = result.routeSubject; + const subjectId = subjectIdForSubject(subject); + const outcome = outcomeFromLiveResult(result); + const authorityLimits = authority.manifestDefaults.executionLimits; + const limits: Record = {}; + for (const key of OBSERVATION_LIMIT_NAMES) { + if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; + } + + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt, + producer: LAB_PRODUCER, + producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, + evidenceLayer: "live_route_compatibility" as const, + scenarioId: caseRecord.id, + scenarioVersion: String(authority.manifestDefaults.version), + scenarioManifestDigest: scenarioDigest, + suiteId: caseRecord.suite, + suiteVersion: String(authority.manifestDefaults.suiteVersion), + suiteManifestDigest: suiteDigest, + fixtureDigests, + subject, + subjectId, + startedAt, + completedAt, + executionMode: "live" as const, + attempt: 1, + limits: { ...limits }, + outcome, + assertions: result.assertionResults.map((a) => ({ + id: a.id, + operator: a.operator, + required: a.required, + passed: a.passed, + expectedSummary: "see_assertion_report", + observedSummary: a.observedSummary.slice(0, 512), + ...(a.reason ? { reason: a.reason } : {}), + })), + ...(caseRecord.expectedFailure + ? { expectedFailure: { ...caseRecord.expectedFailure } } + : {}), + environment: { + runtime: { + platform: process.platform, + arch: process.arch, + bunVersion: process.versions.bun ?? Bun.version, + }, + }, + artifactRefs: artifacts, + ...(result.passed + ? {} + : { + failure: { + class: result.classification, + code: result.secondaryCode ?? result.classification, + retryable: false, + attribution: + result.classification === "harness_failure" + ? ("harness" as const) + : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) + ? ("environment" as const) + : ("route" as const), + }, + }), + }; + + const event = assignEventId(eventWithoutId) as ObservationEvent; + return { event, artifacts }; + } finally { + if (ownsStore) store.close(); + } +} + +export function persistLiveResult( + result: LiveScenarioRunResult, + caseRecord: CaseRecord, + authority: CaseAuthority, + opts: PersistLiveOptions = {}, +): PersistedLiveObservation { + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const { event } = observationFromLiveResult(result, caseRecord, authority, { + ...opts, + artifactStore: store, + }); + appendLabEvent(paths.ledgerPath, event); + return { event, ledgerPath: paths.ledgerPath }; + } finally { + if (ownsStore) store.close(); + } +} diff --git a/src/lab/paths.ts b/src/lab/paths.ts index d54e11f4d..068db3203 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -34,6 +34,11 @@ export function labExportDir(configDir = getConfigDir()): string { return join(labRoot(configDir), "export"); } +/** Opaque per-installation salt for local fingerprinting (never exported as evidence). */ +export function labInstallationSaltPath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "installation-salt.bin"); +} + /** Ensure lab directories exist with restrictive permissions where the platform allows. */ export function ensureLabDirs(configDir = getConfigDir()): { root: string; diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 806a345e0..1b7ea2e88 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,4 +1,4 @@ -import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; +import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1 } from "../events/types"; import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; import type { VerificationRole } from "../conformance/types"; @@ -18,6 +18,7 @@ export interface ScenarioRequirements { inboundProtocols?: string[]; upstreamProtocols?: string[]; surfaces?: string[]; + requiredClaims?: string[]; freshness?: { maxAgeMs: number | null }; } @@ -51,6 +52,25 @@ function scenarioApplicableToRequirements( ); } +/** Applicability for live_route_compatibility using RouteSubjectV1 fields. */ +export function routeSubjectApplicableToRequirements( + requirements: ScenarioRequirements, + subject: RouteSubjectV1, + routeRequiredClaims: string[] = [], +): boolean { + const inbound = requirements.inboundProtocols ?? []; + const upstream = requirements.upstreamProtocols ?? []; + const surfaces = requirements.surfaces ?? []; + const requiredClaims = requirements.requiredClaims ?? []; + const claimsOk = requiredClaims.every((claim) => routeRequiredClaims.includes(claim)); + return ( + inbound.includes(subject.inboundProtocol) && + upstream.includes(subject.upstreamProtocol) && + surfaces.includes(subject.surface) && + claimsOk + ); +} + function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } @@ -79,9 +99,10 @@ function scenarioContractFromManifest( const upstreamProtocols = parseStringArray(row.upstreamProtocols); const surfaces = parseStringArray(row.surfaces); if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; + const requiredClaims = parseStringArray(row.requiredClaims) ?? []; const freshness = parseFreshness(scenarioManifest.freshness); if (!freshness) return null; - return { inboundProtocols, upstreamProtocols, surfaces, freshness }; + return { inboundProtocols, upstreamProtocols, surfaces, requiredClaims, freshness }; } function effectiveMaxAgeMs( @@ -117,7 +138,8 @@ export function evaluateAllApplicableRequiredPassV1( observations: ObservationEvent[], executionMode: ExecutionMode, opts: { - subject?: ProtocolSubjectV1; + subject?: ProtocolSubjectV1 | RouteSubjectV1; + routeRequiredClaims?: string[]; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; asOf?: number; @@ -160,9 +182,12 @@ export function evaluateAllApplicableRequiredPassV1( } } - if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject?.subjectKind === "protocol") { if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; } + if (suiteManifest.evidenceLayer === "live_route_compatibility" && opts.subject?.subjectKind === "route") { + if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeRequiredClaims ?? [])) continue; + } applicableRequired.push(s.id); scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } diff --git a/src/lab/subject/behavior-fingerprint.ts b/src/lab/subject/behavior-fingerprint.ts new file mode 100644 index 000000000..9276d12ac --- /dev/null +++ b/src/lab/subject/behavior-fingerprint.ts @@ -0,0 +1,33 @@ +import { createHash } from "node:crypto"; +import { jcsStringify } from "../digest"; +import type { LabRouteContext } from "../live/types"; + +const CLOSED_BEHAVIOR_KEYS = [ + "wire.adapter", + "wire.inboundProtocol", + "wire.upstreamProtocol", + "wire.surface", + "route.allowPrivateNetwork", + "route.labRunApproval", +] as const; + +/** Build a closed-key behavior fingerprint from lab-forced route context fields. */ +export function buildBehaviorFingerprintV1(routeContext: LabRouteContext): string { + const values: Record = { + "wire.adapter": { source: "lab_forced", value: routeContext.effectiveAdapter }, + "wire.inboundProtocol": { source: "lab_forced", value: routeContext.inboundProtocol }, + "wire.upstreamProtocol": { source: "lab_forced", value: routeContext.upstreamProtocol }, + "wire.surface": { source: "lab_forced", value: routeContext.surface }, + "route.allowPrivateNetwork": { source: "lab_forced", value: routeContext.allowPrivateNetwork === true }, + "route.labRunApproval": { source: "lab_forced", value: routeContext.labRunApproval === true }, + }; + for (const key of CLOSED_BEHAVIOR_KEYS) { + if (!(key in values)) throw new Error(`harness_failure: missing behavior key ${key}`); + } + const payload = { + schemaVersion: 1, + resolverVersion: 1, + values, + }; + return createHash("sha256").update(jcsStringify(payload)).digest("hex"); +} diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts new file mode 100644 index 000000000..c2d5654d2 --- /dev/null +++ b/src/lab/subject/installation-salt.ts @@ -0,0 +1,21 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { labInstallationSaltPath, labRoot } from "../paths"; + +const SALT_BYTES = 32; + +/** Read or create the per-installation salt used for local fingerprinting. */ +export function readInstallationSalt(configDir?: string): Uint8Array { + const path = labInstallationSaltPath(configDir); + if (existsSync(path)) { + const bytes = readFileSync(path); + if (bytes.byteLength !== SALT_BYTES) { + throw new Error("harness_failure: invalid installation salt length"); + } + return new Uint8Array(bytes); + } + mkdirSync(labRoot(configDir), { recursive: true, mode: 0o700 }); + const salt = randomBytes(SALT_BYTES); + writeFileSync(path, salt, { mode: 0o600 }); + return new Uint8Array(salt); +} diff --git a/src/lab/subject/route-subject.ts b/src/lab/subject/route-subject.ts new file mode 100644 index 000000000..9b6c22808 --- /dev/null +++ b/src/lab/subject/route-subject.ts @@ -0,0 +1,67 @@ +import { localFingerprint } from "../digest"; +import type { RouteSubjectV1 } from "../events/types"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; +import { buildBehaviorFingerprintV1 } from "./behavior-fingerprint"; +import { readInstallationSalt } from "./installation-salt"; + +const COMPAT_VERSION = "live-v1"; + +function destinationSnapshotForFingerprint(destination: LabDestinationV1): Record { + return { + scheme: destination.scheme, + host: destination.host, + port: destination.port, + basePath: destination.basePath, + }; +} + +/** Build a frozen RouteSubjectV1 from route context and immutable destination snapshot. */ +export function buildRouteSubjectV1( + routeContext: LabRouteContext, + destination: LabDestinationV1, + configDir?: string, +): RouteSubjectV1 { + const salt = readInstallationSalt(configDir); + const endpointFingerprint = localFingerprint( + "endpoint", + destinationSnapshotForFingerprint(destination), + salt, + ); + const providerInstanceFingerprint = localFingerprint( + "providerInstance", + { + providerId: routeContext.providerId, + clientModelId: routeContext.clientModelId, + upstreamModelId: routeContext.upstreamModelId, + effectiveAdapter: routeContext.effectiveAdapter, + inboundProtocol: routeContext.inboundProtocol, + upstreamProtocol: routeContext.upstreamProtocol, + surface: routeContext.surface, + }, + salt, + ); + return Object.freeze({ + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: routeContext.providerId, + providerInstanceFingerprint, + clientModelId: routeContext.clientModelId, + upstreamModelId: routeContext.upstreamModelId, + effectiveAdapter: routeContext.effectiveAdapter, + inboundProtocol: routeContext.inboundProtocol, + upstreamProtocol: routeContext.upstreamProtocol, + surface: routeContext.surface, + opencodexCompatibilityVersion: COMPAT_VERSION, + behaviorFingerprint: buildBehaviorFingerprintV1(routeContext), + endpointFingerprint, + dependencies: routeContext.dependencies ?? [], + }); +} + +/** Deep-freeze a route subject before live execution. */ +export function freezeRouteSubject(subject: RouteSubjectV1): RouteSubjectV1 { + return Object.freeze({ + ...subject, + dependencies: [...subject.dependencies], + }); +} diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts new file mode 100644 index 000000000..91950f729 --- /dev/null +++ b/tests/lab-live-probe.test.ts @@ -0,0 +1,285 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + CL03_LIVE_SUITES, + createMockTransport, + discoverLiveScenarios, + evaluateAllApplicableRequiredPassV1, + executeInertTool, + expandLiveScenario, + invokeMcpStub, + listInertToolDefinitions, + listLiveScenarioIds, + loadLiveCaseAuthority, + observationFromLiveResult, + persistLiveResult, + rebuildLabProjection, + registerMcpStubTool, + routeSubjectApplicableToRequirements, + runLiveScenario, + runLiveSuite, + scenarioManifestDigest, + subjectIdForSubject, +} from "../src/lab"; +import { replayLabLedger } from "../src/lab/ledger/store"; +import { clearMcpStub } from "../src/lab/live/mcp-loopback"; +import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; +import type { LabRouteContext } from "../src/lab/live/types"; +import type { RouteSubjectV1 } from "../src/lab/events/types"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-probe-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } + delete process.env.OPENCODEX_HOME; + clearMcpStub(); +}); + +function mockRoute(overrides: Partial = {}): LabRouteContext { + return { + providerId: "fixture-provider", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-chat", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-chat", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: ["tools", "image", "reasoning"], + ...overrides, + }; +} + +function transportForCase(caseRecord: ReturnType[number]) { + const entries = []; + if (caseRecord.initiatingRequest) { + entries.push({ + status: 200, + headers: { "content-type": caseRecord.fixture.mediaType }, + body: caseRecord.fixture.bytesUtf8, + }); + } + return createMockTransport({ entries }); +} + +describe("CL-03 live probe harness", () => { + test("loads frozen 10-case live authority with reproducible digests", () => { + const authority = loadLiveCaseAuthority(); + expect(authority.cases.length).toBe(10); + const ids = authority.cases.map((c) => c.id); + expect(ids).toContain("responses-core.live.basic-turn"); + expect(ids).toContain("mcp-core.live.synthetic-tool"); + const expandedA = expandLiveScenario(authority.cases[0]!, authority); + const expandedB = expandLiveScenario(authority.cases[0]!, authority); + expect(scenarioManifestDigest(expandedA)).toBe(scenarioManifestDigest(expandedB)); + }); + + test("discovers all CL-03 live suites", () => { + const authority = loadLiveCaseAuthority(); + const scenarios = discoverLiveScenarios(authority, CL03_LIVE_SUITES); + expect(scenarios.length).toBe(10); + expect(listLiveScenarioIds()).toHaveLength(10); + }); + + test("runs live suite with mock transport and no external network", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const route = mockRoute(); + const summary = await runLiveSuite(route, ["responses-core", "chat-core"], { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ + entries: [ + { status: 200, body: authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!.fixture.bytesUtf8 }, + { status: 200, body: authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!.fixture.bytesUtf8 }, + ], + }), + }); + expect(summary.total).toBe(2); + expect(summary.results.every((r) => r.routeSubject.subjectKind === "route")).toBe(true); + }); + + test("inert tool round-trips validate args and return static results", () => { + expect(listInertToolDefinitions().map((t) => t.name)).toContain("lookup"); + expect(executeInertTool("lookup", { q: "x" }).output).toBe("RESULT"); + expect(() => executeInertTool("lookup", { q: 1 })).toThrow(); + }); + + test("MCP lab stub only — no user mcpServers", () => { + registerMcpStubTool({ namespace: "lab", name: "echo" }); + const result = invokeMcpStub("lab", "echo", { x: 1 }, { content: [{ type: "text", text: "ECHO" }] }); + expect(result.content[0]?.text).toBe("ECHO"); + }); + + test("raw URLs and secrets never appear in persisted live evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: transportForCase(caseRecord), + }); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); + const serialized = JSON.stringify(event); + expect(serialized).not.toContain("api.example.com"); + expect(serialized).not.toContain("https://"); + expect(serialized).not.toContain("apiKey"); + expect(serialized).not.toContain("fixture-key"); + expect(event.evidenceLayer).toBe("live_route_compatibility"); + expect(event.executionMode).toBe("live"); + expect(event.subject.subjectKind).toBe("route"); + }); + + test("JSONL persistence and SQLite rebuild for live evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute({ + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-chat", + surface: "responses-sse", + }), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: transportForCase(caseRecord), + }); + persistLiveResult(result, caseRecord, authority, { configDir: home }); + const projection = rebuildLabProjection(home); + expect(projection.events).toBeGreaterThan(0); + expect(projection.corruptions).toHaveLength(0); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + expect(replay.events.length).toBeGreaterThan(0); + }); + + test("route subject applicability and cross-route evidence reuse prevention", () => { + const authority = loadLiveCaseAuthority(); + const subjectA: RouteSubjectV1 = { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: "provider-a", + providerInstanceFingerprint: "a".repeat(64), + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "live-v1", + behaviorFingerprint: "b".repeat(64), + endpointFingerprint: "c".repeat(64), + dependencies: [], + }; + const subjectB: RouteSubjectV1 = { ...subjectA, providerId: "provider-b", endpointFingerprint: "d".repeat(64) }; + expect(subjectIdForSubject(subjectA)).not.toBe(subjectIdForSubject(subjectB)); + const caseRecord = authority.cases[0]!; + const requirements = caseRecord.requirements; + expect(routeSubjectApplicableToRequirements(requirements, subjectA, [])).toBe(true); + expect(routeSubjectApplicableToRequirements( + { ...requirements, inboundProtocols: ["anthropic-messages"] }, + subjectA, + [], + )).toBe(false); + }); + + test("freshness and contradictory evidence semantics for live suites", () => { + const authority = loadLiveCaseAuthority(); + const suite = expandLiveSuiteManifest("responses-core", authority); + expect(suite.freshness.maxAgeMs).toBe(604800000); + expect(suite.evidenceLayer).toBe("live_route_compatibility"); + const routeSubject: RouteSubjectV1 = { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: "provider-a", + providerInstanceFingerprint: "a".repeat(64), + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "live-v1", + behaviorFingerprint: "b".repeat(64), + endpointFingerprint: "c".repeat(64), + dependencies: [], + }; + const evalEmpty = evaluateAllApplicableRequiredPassV1(suite, [], "live", { + subject: routeSubject, + loadScenarioRequirements: () => ({ + inboundProtocols: ["openai-responses"], + upstreamProtocols: ["openai-responses"], + surfaces: ["responses-http"], + requiredClaims: [], + freshness: { maxAgeMs: 604800000 }, + }), + }); + expect(evalEmpty.canVerify).toBe(false); + expect(evalEmpty.missingRequiredScenarioIds.length).toBeGreaterThan(0); + }); + + test("timeouts classify as blockers not degraded/unsupported", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases[0]!; + const result = await runLiveScenario(caseRecord, mockRoute({ + upstreamProtocol: "openai-responses", + }), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ + entries: [{ status: 200, body: "{}", error: "total_timeout" }], + }), + }); + expect(result.classification).toBe("timeout"); + expect(result.classification).not.toBe("protocol_failure"); + }); + + test("auth failure is a blocker", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases[0]!; + const result = await runLiveScenario(caseRecord, mockRoute({ + upstreamProtocol: "openai-responses", + }), { + configDir: home, + authOutcome: "blocked", + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + }); + expect(result.classification).toBe("authentication_blocked"); + }); + + test("reasoning replay exposes no private material in persisted evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases.find((c) => c.id === "reasoning-core.live.replay")!; + const result = await runLiveScenario(caseRecord, mockRoute({ + upstreamProtocol: "openai-responses", + requiredClaims: ["reasoning"], + }), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ entries: [{ status: 200, body: "{}" }] }), + }); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); + expect(JSON.stringify(event)).not.toContain("PLAN"); + }); +}); diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts new file mode 100644 index 000000000..cbc27efb8 --- /dev/null +++ b/tests/lab-live-sandbox.test.ts @@ -0,0 +1,211 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + assertDestinationAddressSet, + assertHostSniMatch, + assertLeaseScope, + buildRouteSubjectV1, + createCredentialLease, + createLabDestination, + createMockTransport, + createSandboxResourceState, + enforceSandboxLimits, + freezeRouteSubject, + LabCredentialError, + LabDestinationError, + LabSandboxError, + prepareLiveSandbox, + rejectProxyEnvironment, + TransportError, + classifyTransportError, +} from "../src/lab"; +import type { LabRouteContext } from "../src/lab/live/types"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } + delete process.env.OPENCODEX_HOME; +}); + +function baseRoute(overrides: Partial = {}): LabRouteContext { + return { + providerId: "fixture-provider", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-chat", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-chat", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: [], + ...overrides, + }; +} + +describe("CL-03 live sandbox security seams", () => { + test("rejects proxy environment variables", () => { + expect(() => rejectProxyEnvironment({ HTTP_PROXY: "http://evil" })).toThrow(LabSandboxError); + expect(() => rejectProxyEnvironment({ no_proxy: "localhost" })).toThrow(LabSandboxError); + }); + + test("prepareLiveSandbox allows only TZ=UTC and NO_COLOR=1", () => { + const env = prepareLiveSandbox({}); + expect(env).toEqual({ TZ: "UTC", NO_COLOR: "1" }); + }); + + test("DNS/address mismatch fails before credentials", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + expect(() => assertDestinationAddressSet(destination, [{ address: "1.2.3.4", family: 4 }])) + .toThrow(LabDestinationError); + expect(() => createCredentialLease({ destination, authOutcome: "blocked" })) + .toThrow(LabCredentialError); + }); + + test("host/SNI mismatch fails before credentials", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + expect(() => assertHostSniMatch(destination, "evil.example.com")) + .toThrow(LabDestinationError); + }); + + test("metadata and link-local destinations are blocked", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + await expect(createLabDestination({ + baseUrl: "http://169.254.169.254/latest/meta-data", + labRunApproval: true, + configDir: home, + })).rejects.toThrow(LabDestinationError); + }); + + test("private network requires permission and lab approval", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + await expect(createLabDestination({ + baseUrl: "http://127.0.0.1:11434/v1", + configDir: home, + })).rejects.toThrow(LabDestinationError); + const ok = await createLabDestination({ + baseUrl: "http://127.0.0.1:11434/v1", + allowPrivateNetwork: true, + labRunApproval: true, + resolve: async () => [{ address: "127.0.0.1", family: 4 }], + configDir: home, + }); + expect(ok.privateNetwork).toBe(true); + }); + + test("redirects fail closed in mock transport", async () => { + const transport = createMockTransport({ + entries: [{ status: 302, body: "", headers: { location: "https://evil.example.com" } }], + }); + await expect(transport.request({ method: "GET", path: "/" })).rejects.toThrow(TransportError); + }); + + test("credential lease scope is destination-bound", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const lease = createCredentialLease({ destination, budget: 1 }); + assertLeaseScope(lease, destination); + lease.consume(); + expect(() => lease.consume()).toThrow(LabCredentialError); + }); + + test("auth/quota/network/transient classify as blockers not compatibility failures", () => { + expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification) + .toBe("authentication_blocked"); + expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification) + .toBe("quota_blocked"); + expect(classifyTransportError(new TransportError("network_blocked", "net")).classification) + .toBe("network_failure"); + expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification) + .toBe("provider_transient"); + expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification) + .toBe("timeout"); + }); + + test("resource ceilings enforce child process limit", () => { + const state = createSandboxResourceState(); + const limits = { + totalTimeoutMs: 120000, + connectTimeoutMs: 10000, + firstByteTimeoutMs: 30000, + inactivityTimeoutMs: 30000, + maxRequests: 16, + maxInputBytes: 8388608, + maxOutputBytes: 16777216, + maxOutputTokens: 32768, + maxToolCalls: 32, + maxMemoryBytes: 536870912, + maxChildProcesses: 0, + maxArtifacts: 16, + perArtifactBytes: 262144, + aggregateArtifactBytes: 1048576, + }; + expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })) + .toThrow(LabSandboxError); + }); + + test("endpoint fingerprint comes from immutable destination snapshot only", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1/custom", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); + expect(subject.endpointFingerprint).toHaveLength(64); + expect(JSON.stringify(subject)).not.toContain("api.example.com"); + expect(JSON.stringify(subject)).not.toContain("https://"); + }); + + test("route subject stability across credential rotation", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1" }), destination, home)); + createCredentialLease({ destination, budget: 1 }).consume(); + const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1" }), destination, home)); + expect(subjectA).toEqual(subjectB); + }); +}); From 472a1bad60139d02b37476902ab87fa50545eee8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:59:01 +0200 Subject: [PATCH 02/47] docs(lab): record CL-03 PR #1352 and validation state --- .../001_pr_stack_status.md | 37 +++++++++++++++- .../054_cl03_implementation_record.md | 43 +++++++++++++++---- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 39e7adf73..b88a4f4b0 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -22,7 +22,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | | CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | NOT RECORDED | [#1333](https://github.com/lidge-jun/opencodex/pull/1333) | MERGED TO `dev` at `025c37916225dd685d9217e5b40190600f06d278`; POST-MERGE HARDENING [#1343](https://github.com/lidge-jun/opencodex/pull/1343) MERGED at `eee2dab4d1bbacefce56057adad51d734f346702`; FINAL CLOSURE GATE [#1348](https://github.com/lidge-jun/opencodex/pull/1348) | -| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | — | — | IN PROGRESS (implementation) | +| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | `003f7402f49bfe8dd710a7beba52f717051bfadf` | [#1352](https://github.com/lidge-jun/opencodex/pull/1352) | DRAFT PR OPEN (implementation; not accepted) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -156,4 +156,37 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-00: **ACCEPTED** (merged #1286). - CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. - CL-02: **MERGED** via #1333 at `025c37916225dd685d9217e5b40190600f06d278`; post-merge hardening #1343 is also **MERGED** at `eee2dab4d1bbacefce56057adad51d734f346702`; final closure is tracked in #1348. -- CL-03: **IN PROGRESS** on `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`. +- CL-03: **DRAFT PR OPEN** ([#1352](https://github.com/lidge-jun/opencodex/pull/1352)) on + `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`; + implementation head `003f7402f49bfe8dd710a7beba52f717051bfadf`. Not accepted. +- CL-04: **NOT STARTED** (blocked until CL-03 independent acceptance and review reconciliation). + +## CL-03 implementation log (2026-08-09) + +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting/base SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (#1348 merge on `dev`) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) +- **Scope:** bounded live-route probes for `live_route_compatibility`; live manifest + authority; `RouteSubjectV1` builder; `LabDestinationV1` / credential lease sandbox; + inert tool/MCP stubs; live runner/executor; `observe/from-live` persistence; + projection applicability for route subjects. +- **Explicitly out of scope:** CL-04 CLI/API, CL-05 UI, CL-06 profile fields, Fabric, + shadow/automatic probing, production request-path probes. + +### CL-03 validation (local, 2026-08-09) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: 17/17 passed +- `bun test tests/lab-evidence-ledger.test.ts`: 37/41 passed (4 Windows SQLite `EPERM` + flakes in `wipeSqlite`; `rebuild.ts` unchanged vs `upstream/dev` — pre-existing) +- `bun test tests/lab-live-probe.test.ts`: 12/12 passed +- `bun test tests/lab-live-sandbox.test.ts`: 12/12 passed +- `bun run privacy:scan`: passed +- Cross-platform CI on #1352: pending at open time + +### CL-03 blockers + +- Independent acceptance review not performed +- Draft PR review findings not yet reconciled +- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) diff --git a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md index bacfc4abe..cffef1568 100644 --- a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md +++ b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md @@ -1,22 +1,31 @@ # CL-03 implementation record -## Branch +## Branch and revisions -- `feat/cl-03-live-route-probes` -- Starting SHA: `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) ## Scope delivered 1. Live manifest authority (`023_live_v1_manifest_authority.md`, `024_live_v1_cases.json`, runtime copy) 2. Route subject builder (`src/lab/subject/`) -3. Live sandbox (`src/lab/live/`) +3. Live sandbox (`src/lab/live/` — destination, credential lease, transport, executor, runner) 4. Persistence seam (`src/lab/observe/from-live.ts`) -5. Projection applicability (`routeSubjectApplicableToRequirements`) +5. Projection applicability (`routeSubjectApplicableToRequirements` in `verification.ts`) 6. Tests (`tests/lab-live-sandbox.test.ts`, `tests/lab-live-probe.test.ts`) -## Explicitly not started +## Security design summary -- CL-04 CLI/API +- **Network:** immutable in-memory `LabDestinationV1` snapshot; DNS/policy before credentials; + connect only to approved address set; redirects rejected; metadata/link-local forbidden; + private/loopback requires route `allowPrivateNetwork` plus explicit `labRunApproval`; + inherited proxy env vars rejected. +- **Credentials:** opaque `LabCredentialLeaseV1` bound to destination + transport + budget; + no secret bytes in Lab code; auth failure → `authentication_blocked` / `BLOCKED`. +- **Tools/MCP:** inert Lab-authored tools; loopback MCP stub only; no user `mcpServers`. +- **Process:** sandbox env `TZ=UTC`, `NO_COLOR=1` only; no child processes; resource ceilings enforced. ## Frozen live scenarios (10) @@ -31,6 +40,24 @@ - `reasoning-core.live.replay` - `mcp-core.live.synthetic-tool` +## Validation (2026-08-09) + +| Check | Result | +|---|---| +| `bun x tsc --noEmit` | pass | +| `tests/lab-conformance-harness.test.ts` | 17/17 | +| `tests/lab-evidence-ledger.test.ts` | 37/41 (4 Windows SQLite EPERM flakes; pre-existing) | +| `tests/lab-live-probe.test.ts` | 12/12 | +| `tests/lab-live-sandbox.test.ts` | 12/12 | +| `bun run privacy:scan` | pass | + +## Explicitly not started + +- CL-04 Lab CLI or management/read APIs +- CL-05 Compatibility Matrix UI +- CL-06 Routing Profile compatibility policy + ## Acceptance status -Implementation only — not accepted. +Implementation only — **not accepted**. CL-04 remains blocked until independent CL-03 +acceptance and review reconciliation. From 09d26388b46950934b61885a646925e3cde0db77 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:48:44 +0200 Subject: [PATCH 03/47] test(lab): cover CL-03 review blockers --- tests/lab-live-review-regressions.test.ts | 224 ++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 tests/lab-live-review-regressions.test.ts diff --git a/tests/lab-live-review-regressions.test.ts b/tests/lab-live-review-regressions.test.ts new file mode 100644 index 000000000..e75e52825 --- /dev/null +++ b/tests/lab-live-review-regressions.test.ts @@ -0,0 +1,224 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + buildBehaviorFingerprintV1, + buildRouteSubjectV1, + createCredentialLease, + createLabDestination, + createMockTransport, + createPinnedTransport, + expandLiveSuiteManifest, + loadLiveCaseAuthority, + runLiveScenario, + runLiveSuite, +} from "../src/lab"; +import type { + LabBehaviorValues, + LabRouteContext, + LabTransportResponse, + LiveRunConfig, +} from "../src/lab/live/types"; + +const HOMES: string[] = []; +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-live-review-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } + delete process.env.OPENCODEX_HOME; +}); + +function behavior(overrides: Partial = {}): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: "openai-responses" }, + "wire.upstreamProtocol": { source: "lab_forced", value: "openai-responses" }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + ...overrides, + }; +} + +function route(overrides: Partial = {}): LabRouteContext { + return { + providerId: "fixture-provider", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + behaviorValues: behavior(), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: ["tools", "image", "reasoning"], + availableHarnessFeatures: ["live_transport", "inert_tools", "mcp_loopback", "synthetic_image", "reasoning_replay"], + ...overrides, + }; +} + +const LIMITS: LiveRunConfig = { + totalTimeoutMs: 120_000, + connectTimeoutMs: 10_000, + firstByteTimeoutMs: 30_000, + inactivityTimeoutMs: 30_000, + maxRequests: 16, + maxInputBytes: 8 * 1024 * 1024, + maxOutputBytes: 16 * 1024 * 1024, + maxOutputTokens: 32_768, + maxToolCalls: 32, + maxMemoryBytes: 512 * 1024 * 1024, + maxChildProcesses: 0, + maxArtifacts: 16, + perArtifactBytes: 256 * 1024, + aggregateArtifactBytes: 1024 * 1024, +}; + +describe("CL-03 independent-review regressions", () => { + test("live assertions are driven by observed route bytes, not frozen fixture bytes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ entries: [{ status: 200, body: JSON.stringify({ status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "WRONG" }] }] }) }] }), + }); + expect(result.passed).toBe(false); + expect(result.classification).toBe("protocol_failure"); + }); + + test("live execution fails closed when no real route transport is supplied", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + }); + expect(result.passed).toBe(false); + expect(result.classification).toBe("harness_failure"); + expect(result.secondaryCode).toBe("live_transport_required"); + }); + + test("resolved metadata/private addresses are policy-checked, not merely recorded", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + await expect(createLabDestination({ + baseUrl: "https://safe-looking.example/v1", + labRunApproval: true, + resolve: async () => [{ address: "169.254.169.254", family: 4 }], + configDir: home, + })).rejects.toThrow(); + await expect(createLabDestination({ + baseUrl: "https://safe-looking.example/v1", + labRunApproval: true, + resolve: async () => [{ address: "10.0.0.8", family: 4 }], + configDir: home, + })).rejects.toThrow(); + }); + + test("pinned transport uses the approved address and never hostname fetch", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const lease = createCredentialLease({ destination, transportId: "test", budget: 1 }); + const seen: Array<{ address: string; host: string }> = []; + const transport = createPinnedTransport({ + destination, + lease, + transportId: "test", + limits: LIMITS, + sender: async (_lease, dest, pinned): Promise => { + seen.push({ address: pinned.address, host: dest.host }); + return { status: 200, headers: { "content-type": "application/json" }, body: "{}" }; + }, + }); + await transport.request({ method: "POST", path: "/responses", body: "{}" }); + expect(seen).toEqual([{ address: "93.184.216.34", host: "api.example.com" }]); + }); + + test("behavior fingerprint rejects unknown inputs and changes for effective behavior", () => { + const base = behavior(); + const first = buildBehaviorFingerprintV1(base); + const second = buildBehaviorFingerprintV1({ ...base, "runtime.fastMode": { source: "lab_forced", value: true } }); + expect(first).not.toBe(second); + expect(() => buildBehaviorFingerprintV1({ ...base, "secret.apiKey": { source: "provider_config", value: "nope" } } as LabBehaviorValues)).toThrow(); + }); + + test("transport blockers retain the exact already-built route subject", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ entries: [{ status: 401, body: "" }] }), + }); + expect(result.classification).toBe("authentication_blocked"); + expect(result.routeSubject?.endpointFingerprint).not.toBe("0".repeat(64)); + }); + + test("runLiveSuite does not execute inapplicable route scenarios", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + let requests = 0; + const summary = await runLiveSuite(route(), ["responses-core", "chat-core"], { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ + onRequest: () => { requests += 1; }, + entries: [{ status: 200, body: JSON.stringify({ status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] }) }], + }), + }); + expect(summary.total).toBe(1); + expect(requests).toBe(1); + }); + + test("codex live verification manifest includes the responses basic-turn prerequisite", () => { + const authority = loadLiveCaseAuthority(); + const suite = expandLiveSuiteManifest("codex-core", authority); + expect(suite.scenarios.map((s) => s.id)).toContain("responses-core.live.basic-turn"); + }); + + test("route subject uses configured instance identity and supplied compatibility version", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const a = buildRouteSubjectV1(route({ providerInstanceKey: "instance-a" }), destination, home); + const b = buildRouteSubjectV1(route({ providerInstanceKey: "instance-b" }), destination, home); + expect(a.providerInstanceFingerprint).not.toBe(b.providerInstanceFingerprint); + expect(a.opencodexCompatibilityVersion).toBe("a".repeat(64)); + }); +}); From 38e7824d6375c9a87a8b95e31b7c89a4d5c49dc2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:54:49 +0200 Subject: [PATCH 04/47] fix(lab): make CL-03 live evidence fail closed --- src/lab/live/credential-lease.ts | 63 ++-- src/lab/live/destination.ts | 155 +++++----- src/lab/live/executor.ts | 390 ++++++++++-------------- src/lab/live/manifest.ts | 121 +++----- src/lab/live/runner.ts | 29 +- src/lab/live/sandbox.ts | 103 ++----- src/lab/live/suite-manifest.ts | 70 ++--- src/lab/live/transport.ts | 166 +++++----- src/lab/live/types.ts | 79 ++++- src/lab/observe/from-live.ts | 253 +++------------ src/lab/subject/behavior-fingerprint.ts | 94 ++++-- src/lab/subject/route-subject.ts | 84 ++--- src/lib/lab-live-pinned-sender.ts | 29 ++ 13 files changed, 687 insertions(+), 949 deletions(-) create mode 100644 src/lib/lab-live-pinned-sender.ts diff --git a/src/lab/live/credential-lease.ts b/src/lab/live/credential-lease.ts index 9934de64e..7869ff490 100644 --- a/src/lab/live/credential-lease.ts +++ b/src/lab/live/credential-lease.ts @@ -2,58 +2,51 @@ import { LAB_CREDENTIAL_LEASE, type LabCredentialLeaseV1, type LabDestinationV1 export class LabCredentialError extends Error { override readonly name = "LabCredentialError"; - constructor( - message: string, - readonly code: "auth_blocked" | "harness_failure", - ) { - super(message); - } + constructor(message: string, readonly code: "harness_failure" | "budget_exhausted") { super(message); } } +interface LeaseState { destinationFingerprint: string; transportId: string; remaining: number } +const LEASE_STATE = new WeakMap(); + export interface CreateLeaseOptions { destination: LabDestinationV1; transportId?: string; budget?: number; - authOutcome?: "ok" | "blocked"; } -/** Create an opaque credential lease bound to destination + transport + budget. */ +/** Trusted credential owner may attach its own secret state to this opaque capability. */ export function createCredentialLease(opts: CreateLeaseOptions): LabCredentialLeaseV1 { - if (opts.authOutcome === "blocked") { - throw new LabCredentialError("credential rejected for destination", "auth_blocked"); - } const budget = opts.budget ?? 1; - let remaining = budget; - const lease: LabCredentialLeaseV1 = { - [LAB_CREDENTIAL_LEASE]: true, - destinationFingerprint: opts.destination.fingerprint, - transportId: opts.transportId ?? "default", - get remainingRequests() { - return remaining; - }, + if (!Number.isInteger(budget) || budget <= 0) throw new LabCredentialError("invalid credential lease budget", "harness_failure"); + const lease = { + [LAB_CREDENTIAL_LEASE]: true as const, + get remainingRequests() { return LEASE_STATE.get(lease)?.remaining ?? 0; }, consume() { - if (remaining <= 0) { - throw new LabCredentialError("credential lease exhausted", "auth_blocked"); - } - remaining -= 1; + const state = LEASE_STATE.get(lease); + if (!state) throw new LabCredentialError("unknown credential lease", "harness_failure"); + if (state.remaining <= 0) throw new LabCredentialError("credential lease exhausted", "budget_exhausted"); + state.remaining -= 1; }, - }; + toString() { return "[LabCredentialLeaseV1]"; }, + toJSON() { throw new LabCredentialError("credential lease is non-serializable", "harness_failure"); }, + } satisfies LabCredentialLeaseV1 & { toString(): string; toJSON(): never }; + LEASE_STATE.set(lease, { + destinationFingerprint: opts.destination.fingerprint, + transportId: opts.transportId ?? "default", + remaining: budget, + }); return Object.freeze(lease); } export function isCredentialLease(value: unknown): value is LabCredentialLeaseV1 { - return typeof value === "object" && value !== null && (value as LabCredentialLeaseV1)[LAB_CREDENTIAL_LEASE] === true; + return typeof value === "object" && value !== null && LEASE_STATE.has(value as object) + && (value as LabCredentialLeaseV1)[LAB_CREDENTIAL_LEASE] === true; } -export function assertLeaseScope( - lease: LabCredentialLeaseV1, - destination: LabDestinationV1, - transportId = "default", -): void { - if (lease.destinationFingerprint !== destination.fingerprint) { - throw new LabCredentialError("lease destination mismatch", "auth_blocked"); - } - if (lease.transportId !== transportId) { - throw new LabCredentialError("lease transport mismatch", "auth_blocked"); +export function assertLeaseScope(lease: LabCredentialLeaseV1, destination: LabDestinationV1, transportId = "default"): void { + const state = LEASE_STATE.get(lease as object); + if (!state) throw new LabCredentialError("unknown credential lease", "harness_failure"); + if (state.destinationFingerprint !== destination.fingerprint || state.transportId !== transportId) { + throw new LabCredentialError("credential lease scope mismatch", "harness_failure"); } } diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts index 465f79b63..6f1bfd723 100644 --- a/src/lab/live/destination.ts +++ b/src/lab/live/destination.ts @@ -1,35 +1,50 @@ -import { assessUrlDestination } from "../../lib/destination-policy"; +import { isIP } from "node:net"; +import { assessUrlDestination, resolvePublicAddresses } from "../../lib/destination-policy"; import { localFingerprint } from "../digest"; import { readInstallationSalt } from "../subject/installation-salt"; import type { DnsResolver, LabDestinationV1 } from "./types"; export class LabDestinationError extends Error { override readonly name = "LabDestinationError"; - constructor( - message: string, - readonly code: string, - ) { - super(message); - } + constructor(message: string, readonly code: string) { super(message); } } function normalizeBasePath(pathname: string): string { if (!pathname || pathname === "/") return ""; - return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname; + const trimmed = pathname.endsWith("/") ? pathname.slice(0, -1) : pathname; + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; } -function destinationFingerprintValue(snapshot: { - scheme: string; - host: string; - port: number; - basePath: string; -}): Record { - return { - scheme: snapshot.scheme, - host: snapshot.host, - port: snapshot.port, - basePath: snapshot.basePath, - }; +function addressUrl(scheme: "http" | "https", address: string): string { + return `${scheme}://${isIP(address) === 6 ? `[${address}]` : address}/`; +} + +function validateResolvedAddress( + scheme: "http" | "https", + address: { address: string; family: 4 | 6 }, + allowPrivateNetwork: boolean, + labRunApproval: boolean, +): boolean { + const actualFamily = isIP(address.address); + if ((actualFamily !== 4 && actualFamily !== 6) || actualFamily !== address.family) { + throw new LabDestinationError("resolver returned an invalid address/family", "harness_failure"); + } + const assessment = assessUrlDestination(addressUrl(scheme, address.address)); + if (!assessment) throw new LabDestinationError("resolver address could not be classified", "harness_failure"); + if (["metadata", "link-local", "unspecified"].includes(assessment.kind)) { + throw new LabDestinationError(`blocked resolved destination: ${assessment.detail}`, "harness_failure"); + } + const privateAnswer = ["private", "loopback", "localhost"].includes(assessment.kind); + if (privateAnswer && (!allowPrivateNetwork || !labRunApproval)) { + throw new LabDestinationError("resolved private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); + } + return privateAnswer; +} + +function canonicalAddresses(addresses: Array<{ address: string; family: 4 | 6 }>): Array<{ address: string; family: 4 | 6 }> { + const unique = new Map(); + for (const row of addresses) unique.set(`${row.family}:${row.address}`, { ...row }); + return [...unique.values()].sort((a, b) => a.family - b.family || (a.address < b.address ? -1 : a.address > b.address ? 1 : 0)); } export interface CreateDestinationOptions { @@ -40,78 +55,72 @@ export interface CreateDestinationOptions { configDir?: string; } -/** Create an immutable destination snapshot from provider baseUrl with DNS policy checks. */ -export async function createLabDestination( - opts: CreateDestinationOptions, -): Promise { +/** Resolve once, policy-check every answer, then freeze the exact snapshot used by all later stages. */ +export async function createLabDestination(opts: CreateDestinationOptions): Promise { let parsed: URL; - try { - parsed = new URL(opts.baseUrl); - } catch { - throw new LabDestinationError("invalid provider baseUrl", "harness_failure"); - } - if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { - throw new LabDestinationError("unsupported URL scheme", "harness_failure"); + try { parsed = new URL(opts.baseUrl); } catch { throw new LabDestinationError("invalid provider baseUrl", "harness_failure"); } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new LabDestinationError("unsupported URL scheme", "harness_failure"); + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new LabDestinationError("provider destination may not contain userinfo, query, or fragment", "harness_failure"); } - const assessment = assessUrlDestination(parsed.toString()); - if (assessment?.kind === "metadata" || assessment?.kind === "link-local" || assessment?.kind === "unspecified") { - throw new LabDestinationError(`blocked destination: ${assessment.detail}`, "harness_failure"); + const scheme = parsed.protocol === "https:" ? "https" as const : "http" as const; + const literal = assessUrlDestination(parsed.toString()); + if (literal && ["metadata", "link-local", "unspecified"].includes(literal.kind)) { + throw new LabDestinationError(`blocked destination: ${literal.detail}`, "harness_failure"); } - const privateNetwork = assessment?.kind === "private" || assessment?.kind === "loopback" || assessment?.kind === "localhost"; - if (privateNetwork && (!opts.allowPrivateNetwork || !opts.labRunApproval)) { + const allowPrivate = opts.allowPrivateNetwork === true; + const approved = opts.labRunApproval === true; + const literalPrivate = Boolean(literal && ["private", "loopback", "localhost"].includes(literal.kind)); + if (literalPrivate && (!allowPrivate || !approved)) { throw new LabDestinationError("private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); } - const resolve = opts.resolve ?? defaultResolver; - const addresses = await resolve(parsed.hostname); - if (addresses.length === 0) { - throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); + let addresses: Array<{ address: string; family: 4 | 6 }>; + let privateNetwork = literalPrivate; + if (opts.resolve) { + try { addresses = await opts.resolve(parsed.hostname); } + catch { throw new LabDestinationError("DNS resolution failed", "network_blocked"); } + if (addresses.length === 0) throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); + for (const row of addresses) privateNetwork = validateResolvedAddress(scheme, row, allowPrivate, approved) || privateNetwork; + } else { + try { + const resolved = await resolvePublicAddresses(parsed.toString(), { + context: "Lab provider destination", + allowPrivateNetwork: allowPrivate && approved, + }); + addresses = resolved.addresses.map((row) => ({ address: row.address, family: row.family === 6 ? 6 as const : 4 as const })); + privateNetwork = resolved.privateNetwork || privateNetwork; + } catch (error) { + throw new LabDestinationError(error instanceof Error ? error.message : "DNS destination policy failed", "network_blocked"); + } } - - const port = parsed.port - ? Number(parsed.port) - : parsed.protocol === "https:" ? 443 : 80; + const frozenAddresses = Object.freeze(canonicalAddresses(addresses).map((row) => Object.freeze(row))); + const port = parsed.port ? Number(parsed.port) : scheme === "https" ? 443 : 80; + if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new LabDestinationError("invalid destination port", "harness_failure"); const snapshot = { - scheme: parsed.protocol === "https:" ? "https" as const : "http" as const, + scheme, host: parsed.hostname.toLowerCase(), port, basePath: normalizeBasePath(parsed.pathname), sniHost: parsed.hostname.toLowerCase(), - addresses: Object.freeze(addresses.map((a) => Object.freeze({ ...a }))), + addresses: frozenAddresses, privateNetwork, }; const salt = readInstallationSalt(opts.configDir); - const fingerprint = localFingerprint("endpoint", destinationFingerprintValue(snapshot), salt); + const fingerprint = localFingerprint("endpoint", { + scheme: snapshot.scheme, host: snapshot.host, port: snapshot.port, basePath: snapshot.basePath, + }, salt); return Object.freeze({ ...snapshot, fingerprint }); } -/** Fail closed when a re-resolution does not match the immutable snapshot. */ -export function assertDestinationAddressSet( - destination: LabDestinationV1, - addresses: Array<{ address: string; family: 4 | 6 }>, -): void { - const a = [...destination.addresses].map((x) => `${x.family}:${x.address}`).sort().join(","); - const b = [...addresses].map((x) => `${x.family}:${x.address}`).sort().join(","); - if (a !== b) { - throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); - } +/** Detect any attempted replacement/re-resolution of the approved immutable address set. */ +export function assertDestinationAddressSet(destination: LabDestinationV1, addresses: Array<{ address: string; family: 4 | 6 }>): void { + const a = destination.addresses.map((x) => `${x.family}:${x.address}`).sort().join(","); + const b = canonicalAddresses(addresses).map((x) => `${x.family}:${x.address}`).join(","); + if (a !== b) throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); } export function assertHostSniMatch(destination: LabDestinationV1, host: string, sni?: string): void { - const normalized = host.toLowerCase(); - if (normalized !== destination.host) { - throw new LabDestinationError("host mismatch", "host_sni_mismatch"); - } - if (sni !== undefined && sni.toLowerCase() !== destination.sniHost) { - throw new LabDestinationError("SNI mismatch", "host_sni_mismatch"); - } -} - -async function defaultResolver(hostname: string): Promise> { - const { lookup } = await import("node:dns/promises"); - const results = await lookup(hostname, { all: true, verbatim: true }); - return results.map((r) => ({ - address: r.address, - family: r.family === 6 ? 6 as const : 4 as const, - })); + if (host.toLowerCase() !== destination.host) throw new LabDestinationError("host mismatch", "host_sni_mismatch"); + if (sni !== undefined && sni.toLowerCase() !== destination.sniHost) throw new LabDestinationError("SNI mismatch", "host_sni_mismatch"); } diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index 2385d9b2e..24ff020ed 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -1,276 +1,202 @@ import { evaluateAssertions } from "../conformance/assertion"; -import { executeScenario } from "../conformance/executor"; -import type { CaseRecord, FailureClassification, FailureRule } from "../conformance/types"; +import { emptyObservation, finalizeObservation, setClientResponse } from "../conformance/observation"; +import { normalizeSseBytes } from "../conformance/sse-normalize"; +import type { CaseRecord, FailureClassification, FailureRule, NormalizedEvent, NormalizedObservation } from "../conformance/types"; import type { RouteSubjectV1 } from "../events/types"; -import { createCredentialLease, LabCredentialError } from "./credential-lease"; -import { createLabDestination } from "./destination"; +import { buildRouteSubjectV1 } from "../subject/route-subject"; +import { createLabDestination, LabDestinationError } from "./destination"; import { expandLiveScenario } from "./manifest"; -import { buildRouteSubjectV1, freezeRouteSubject } from "../subject/route-subject"; -import { - createSandboxResourceState, - enforceSandboxLimits, - prepareLiveSandbox, -} from "./sandbox"; -import { classifyTransportError, createMockTransport, TransportError } from "./transport"; -import type { - DnsResolver, - LabRouteContext, - LabTransport, - LiveRunConfig, - LiveScenarioRunResult, -} from "./types"; +import { createSandboxResourceState, enforceSandboxLimits, LabSandboxError, prepareLiveSandbox } from "./sandbox"; +import { classifyTransportError, TransportError } from "./transport"; +import type { DnsResolver, LabRouteContext, LabRouteExecutor, LabTransport, LiveRunConfig, LiveScenarioRunResult } from "./types"; export interface LiveExecutorOptions { + /** Trusted exact-route executor for production use; owns adapter + credential plumbing. */ + routeExecutor?: LabRouteExecutor; + /** Injectable byte transport used by deterministic tests. Never created implicitly. */ transport?: LabTransport; resolve?: DnsResolver; configDir?: string; - authOutcome?: "ok" | "blocked"; env?: NodeJS.ProcessEnv; } -function liveLimitsFromAuthority(caseRecord: CaseRecord, authorityLimits: Record): LiveRunConfig { +function liveLimitsFromAuthority(authorityLimits: Record): LiveRunConfig { return { - totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, - connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, - firstByteTimeoutMs: authorityLimits.firstByteTimeoutMs ?? 30_000, - inactivityTimeoutMs: authorityLimits.inactivityTimeoutMs ?? 30_000, - maxRequests: authorityLimits.maxRequests ?? 16, - maxInputBytes: authorityLimits.maxInputBytes ?? 8 * 1024 * 1024, - maxOutputBytes: authorityLimits.maxOutputBytes ?? 16 * 1024 * 1024, - maxOutputTokens: authorityLimits.maxOutputTokens ?? 32_768, - maxToolCalls: authorityLimits.maxToolCalls ?? 32, - maxMemoryBytes: authorityLimits.maxMemoryBytes ?? 512 * 1024 * 1024, - maxChildProcesses: authorityLimits.maxChildProcesses ?? 0, - maxArtifacts: authorityLimits.maxArtifacts ?? 16, - perArtifactBytes: authorityLimits.perArtifactBytes ?? 256 * 1024, - aggregateArtifactBytes: authorityLimits.aggregateArtifactBytes ?? 1024 * 1024, + totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, + firstByteTimeoutMs: authorityLimits.firstByteTimeoutMs ?? 30_000, inactivityTimeoutMs: authorityLimits.inactivityTimeoutMs ?? 30_000, + maxRequests: authorityLimits.maxRequests ?? 16, maxInputBytes: authorityLimits.maxInputBytes ?? 8 * 1024 * 1024, + maxOutputBytes: authorityLimits.maxOutputBytes ?? 16 * 1024 * 1024, maxOutputTokens: authorityLimits.maxOutputTokens ?? 32_768, + maxToolCalls: authorityLimits.maxToolCalls ?? 32, maxMemoryBytes: authorityLimits.maxMemoryBytes ?? 512 * 1024 * 1024, + maxChildProcesses: authorityLimits.maxChildProcesses ?? 0, maxArtifacts: authorityLimits.maxArtifacts ?? 16, + perArtifactBytes: authorityLimits.perArtifactBytes ?? 256 * 1024, aggregateArtifactBytes: authorityLimits.aggregateArtifactBytes ?? 1024 * 1024, }; } -function checkRoutePreconditions(routeContext: LabRouteContext, caseRecord: CaseRecord): string | null { +export function isLiveCaseApplicableToRoute(caseRecord: CaseRecord, route: LabRouteContext): boolean { + const req = caseRecord.requirements; + if (!req.inboundProtocols.includes(route.inboundProtocol) || !req.upstreamProtocols.includes(route.upstreamProtocol) || !req.surfaces.includes(route.surface)) return false; + if (req.platforms.length > 0 && !req.platforms.includes(process.platform)) return false; + if (!req.requiredClaims.every((claim) => (route.requiredClaims ?? []).includes(claim))) return false; + if (!req.requiredHarnessFeatures.every((feature) => (route.availableHarnessFeatures ?? []).includes(feature))) return false; + return true; +} + +function routePreconditionFailure(route: LabRouteContext, caseRecord: CaseRecord): string | null { for (const precondition of caseRecord.requirements.routePreconditions) { - if (precondition === "lab_run_approval" && routeContext.labRunApproval !== true) { - return "route_precondition_unmet:lab_run_approval"; + if (precondition === "lab_run_approval" && route.labRunApproval !== true) return "route_precondition_unmet:lab_run_approval"; + if (precondition === "allow_private_network" && route.allowPrivateNetwork !== true) return "route_precondition_unmet:allow_private_network"; + } + return null; +} + +function classifyWithFailureRules(rules: FailureRule[], signal: string): { classification: FailureClassification; secondaryCode: string } { + const rule = rules.find((row) => row.match.includes(signal)); + return rule ? { classification: rule.classification, secondaryCode: rule.secondaryCode ?? signal } + : { classification: "inconclusive", secondaryCode: "unclassified" }; +} + +function pathForProtocol(protocol: string): string { + if (protocol === "openai-chat") return "/chat/completions"; + if (protocol === "anthropic-messages") return "/messages"; + return "/responses"; +} + +function chatObservation(body: string, status: number): NormalizedObservation { + const observation = emptyObservation(); + const output: unknown[] = []; + let text = ""; + let terminal = false; + const trimmed = body.trimStart(); + if (trimmed.startsWith("data:")) { + const toolParts = new Map(); + for (const frame of body.split(/\r?\n\r?\n/)) { + const line = frame.split(/\r?\n/).find((row) => row.startsWith("data:")); + if (!line) continue; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") { if (payload === "[DONE]") terminal = true; continue; } + let json: any; + try { json = JSON.parse(payload); } catch { continue; } + for (const choice of Array.isArray(json.choices) ? json.choices : []) { + if (typeof choice?.delta?.content === "string") text += choice.delta.content; + if (choice?.finish_reason != null) terminal = true; + for (const call of Array.isArray(choice?.delta?.tool_calls) ? choice.delta.tool_calls : []) { + const idx = Number.isInteger(call?.index) ? call.index : toolParts.size; + const prior = toolParts.get(idx) ?? { id: "", name: "", arguments: "" }; + if (typeof call?.id === "string") prior.id = call.id; + if (typeof call?.function?.name === "string") prior.name = call.function.name; + if (typeof call?.function?.arguments === "string") prior.arguments += call.function.arguments; + toolParts.set(idx, prior); + } + } } - if (precondition === "allow_private_network" && routeContext.allowPrivateNetwork !== true) { - return "route_precondition_unmet:allow_private_network"; + for (const row of [...toolParts.entries()].sort((a, b) => a[0] - b[0]).map(([, value]) => value)) { + output.push({ type: "function_call", call_id: row.id, name: row.name, arguments: row.arguments }); } - } - for (const claim of caseRecord.requirements.requiredClaims) { - if (!(routeContext.requiredClaims ?? []).includes(claim)) { - return `required_claim_missing:${claim}`; + } else { + const json = JSON.parse(body) as any; + const choice = Array.isArray(json.choices) ? json.choices[0] : undefined; + if (typeof choice?.message?.content === "string") text = choice.message.content; + terminal = choice?.finish_reason != null; + for (const call of Array.isArray(choice?.message?.tool_calls) ? choice.message.tool_calls : []) { + output.push({ type: "function_call", call_id: call.id, name: call.function?.name, arguments: call.function?.arguments }); } } - return null; + if (text) output.unshift({ type: "message", content: [{ type: "output_text", text }] }); + const json = { status: terminal ? "completed" : "incomplete", output }; + const events: NormalizedEvent[] = terminal ? [{ event: "response.completed", data: { response: json }, ordinal: 0 }] : []; + finalizeObservation(observation, events, json, status); + return observation; } -function transportFromFixtures(caseRecord: CaseRecord): LabTransport { - const entries = []; - if (caseRecord.initiatingRequest && caseRecord.fixture.role === "upstream_response") { - entries.push({ - status: 200, - headers: { "content-type": caseRecord.fixture.mediaType }, - body: caseRecord.fixture.bytesUtf8, - }); - } else if (caseRecord.fixture.role === "upstream_response") { - entries.push({ - status: 200, - headers: { "content-type": caseRecord.fixture.mediaType }, - body: caseRecord.fixture.bytesUtf8, - }); - } else if (caseRecord.fixture.role === "adapter_vector" || caseRecord.fixture.role === "client_request") { - entries.push({ status: 200, body: caseRecord.fixture.bytesUtf8 }); - } else if (caseRecord.fixture.role === "synthetic_tool") { - entries.push({ status: 200, body: "{}" }); +function responsesObservation(body: string, status: number): NormalizedObservation { + const observation = emptyObservation(); + const trimmed = body.trimStart(); + if (trimmed.startsWith("data:") || trimmed.startsWith("event:")) { + const events = normalizeSseBytes(new TextEncoder().encode(body), "openai-responses"); + finalizeObservation(observation, events, null, status); + return observation; } - return createMockTransport({ entries }); + const json = JSON.parse(body) as Record; + finalizeObservation(observation, [], json, status); + const state = typeof json.status === "string" ? json.status : undefined; + if (state === "completed" || state === "failed" || state === "incomplete") { + setClientResponse(observation, { terminal: state === "completed" ? "completed" : state }); + } + return observation; } -function classifyWithFailureRules( - rules: FailureRule[], - signal: string, - defaultClass: FailureClassification, - defaultCode: string, -): { classification: FailureClassification; secondaryCode: string } { - for (const rule of rules) { - if (rule.match.includes(signal) || rule.match.includes("no_prior_rule")) { - if (rule.match.includes("no_prior_rule") && signal !== "no_prior_rule") continue; - if (!rule.match.includes(signal) && !rule.match.includes("no_prior_rule")) continue; - return { - classification: rule.classification, - secondaryCode: rule.secondaryCode ?? defaultCode, - }; - } +function normalizeTransportObservation(caseRecord: CaseRecord, route: LabRouteContext, response: { status: number; body: string }): NormalizedObservation { + const observation = route.upstreamProtocol === "openai-chat" ? chatObservation(response.body, response.status) : responsesObservation(response.body, response.status); + if (route.surface.startsWith("anthropic-") && observation.client.response.terminal === "completed") { + setClientResponse(observation, { terminal: "message_stop" }); } - return { classification: defaultClass, secondaryCode: defaultCode }; + return observation; } -async function executeLiveObservation( - caseRecord: CaseRecord, - transport: LabTransport, - limits: LiveRunConfig, -): Promise<{ observation: Awaited> }> { - const state = createSandboxResourceState(); - if (caseRecord.fixture.role !== "adapter_vector" && caseRecord.fixture.role !== "synthetic_tool") { - const response = await transport.request({ - method: "POST", - path: "/chat/completions", - body: caseRecord.initiatingRequest?.bytesUtf8 ?? caseRecord.fixture.bytesUtf8, - }); - enforceSandboxLimits(state, limits, { - requests: 1, - inputBytes: (caseRecord.initiatingRequest?.bytesUtf8 ?? caseRecord.fixture.bytesUtf8).length, - outputBytes: response.body.length, - }); - } - const observation = await executeScenario(caseRecord); - return { observation }; +async function withTotalTimeout(timeoutMs: number, run: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live scenario total timeout")), timeoutMs); + try { + return await Promise.race([ + run(controller.signal), + new Promise((_, reject) => controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true })), + ]); + } finally { clearTimeout(timer); } } -/** Run one live scenario against a route context with injectable transport. */ -export async function runLiveScenario( - caseRecord: CaseRecord, - routeContext: LabRouteContext, - opts: LiveExecutorOptions = {}, -): Promise { +export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabRouteContext, opts: LiveExecutorOptions = {}): Promise { const diagnostics: string[] = []; const startedAt = Date.now(); - const complete = ( - partial: Omit, - ): LiveScenarioRunResult => ({ - ...partial, - startedAt, - completedAt: Math.max(startedAt, Date.now()), - }); - + let routeSubject: RouteSubjectV1 | undefined; + const complete = (partial: Omit): LiveScenarioRunResult => ({ ...partial, startedAt, completedAt: Math.max(startedAt, Date.now()) }); try { - prepareLiveSandbox(opts.env); - const destination = await createLabDestination({ - baseUrl: routeContext.baseUrl, - allowPrivateNetwork: routeContext.allowPrivateNetwork, - labRunApproval: routeContext.labRunApproval, - resolve: opts.resolve, - configDir: opts.configDir, - }); - const routeSubject = freezeRouteSubject( - buildRouteSubjectV1(routeContext, destination, opts.configDir), - ); - const preconditionFailure = checkRoutePreconditions(routeContext, caseRecord); + const environment = prepareLiveSandbox(opts.env); + if (!isLiveCaseApplicableToRoute(caseRecord, routeContext)) { + return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: "inconclusive", secondaryCode: "scenario_inapplicable", assertionResults: [], diagnostics, routeSubject }); + } + const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); + routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); + const preconditionFailure = routePreconditionFailure(routeContext, caseRecord); if (preconditionFailure) { - return complete({ - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed: false, - classification: "inconclusive", - secondaryCode: preconditionFailure, - assertionResults: [], - diagnostics: [preconditionFailure], - routeSubject, - }); + return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: "inconclusive", secondaryCode: preconditionFailure, assertionResults: [], diagnostics: [preconditionFailure], routeSubject }); } - - const lease = createCredentialLease({ - destination, - authOutcome: opts.authOutcome ?? "ok", - }); - void lease; - const authority = await import("./manifest").then((m) => m.loadLiveCaseAuthority()); const expanded = expandLiveScenario(caseRecord, authority); - const limits = liveLimitsFromAuthority(caseRecord, expanded.executionLimits as Record); - const transport = opts.transport ?? transportFromFixtures(caseRecord); - - const { observation } = await executeLiveObservation(caseRecord, transport, limits); + const limits = liveLimitsFromAuthority(expanded.executionLimits as Record); + const state = createSandboxResourceState(); + let observation: NormalizedObservation; + if (opts.routeExecutor) { + observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!({ routeContext, destination, routeSubject: routeSubject!, scenarioId: caseRecord.id, initiatingRequest: caseRecord.initiatingRequest?.bytesUtf8, limits, signal, environment })); + } else if (opts.transport && caseRecord.initiatingRequest) { + const body = caseRecord.initiatingRequest.bytesUtf8; + const inputBytes = new TextEncoder().encode(body).byteLength; + enforceSandboxLimits(state, limits, { requests: 1, inputBytes }); + const response = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.transport!.request({ method: "POST", path: pathForProtocol(routeContext.upstreamProtocol), body, signal })); + if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`); + if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429"); + if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); + if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`); + const outputBytes = new TextEncoder().encode(response.body).byteLength; + enforceSandboxLimits(state, limits, { outputBytes, outputTokens: Math.ceil(outputBytes / 4), peakMemoryBytes: process.memoryUsage().rss }); + observation = normalizeTransportObservation(caseRecord, routeContext, response); + } else { + throw new TransportError("harness_failure", caseRecord.initiatingRequest ? "live_transport_required" : "live_route_executor_required"); + } const assertionResults = evaluateAssertions(caseRecord.assertions, observation); - const requiredFailures = assertionResults.filter((r) => r.required && !r.passed); - + const requiredFailures = assertionResults.filter((row) => row.required && !row.passed); + const failureRules = expanded.failureRules as FailureRule[]; if (caseRecord.expectedFailure) { - const listed = caseRecord.expectedFailure.assertionIds; - const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed === true); - const expectedFailureMatched = controlPassed && requiredFailures.length === 0; - return complete({ - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed: expectedFailureMatched, - classification: expectedFailureMatched - ? caseRecord.expectedFailure.expectedClass as FailureClassification - : "protocol_failure", - secondaryCode: expectedFailureMatched - ? caseRecord.expectedFailure.expectedCode - : "deterministic_assertion", - assertionResults, - diagnostics, - routeSubject, - }); + const matched = caseRecord.expectedFailure.assertionIds.every((id) => assertionResults.find((row) => row.id === id)?.passed === true) && requiredFailures.length === 0; + return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: matched, classification: matched ? caseRecord.expectedFailure.expectedClass : "protocol_failure", secondaryCode: matched ? caseRecord.expectedFailure.expectedCode : "deterministic_assertion", assertionResults, diagnostics, routeSubject }); } - const passed = requiredFailures.length === 0; - const failureRules = expanded.failureRules as FailureRule[]; - const signal = passed ? "pass" : "required_assertion_failed"; - const classified = passed - ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass" } - : classifyWithFailureRules(failureRules, signal, "protocol_failure", "deterministic_assertion"); - - return complete({ - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed, - classification: passed ? "inconclusive" : classified.classification, - secondaryCode: passed ? undefined : classified.secondaryCode, - assertionResults, - diagnostics, - routeSubject, - }); + const classified = passed ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass" } : classifyWithFailureRules(failureRules, "required_assertion_failed"); + return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }); } catch (error) { - diagnostics.push(String(error)); - if (error instanceof LabCredentialError || (error instanceof Error && error.name === "LabCredentialError")) { - const credError = error as LabCredentialError; - return complete({ - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed: false, - classification: "authentication_blocked", - secondaryCode: credError.code, - assertionResults: [], - diagnostics, - routeSubject: fallbackRouteSubject(routeContext), - transportError: "auth_blocked", - }); - } - const transportCode = error instanceof TransportError - || (error instanceof Error && error.name === "TransportError") - ? (error as TransportError).code - : undefined; - const classified = classifyTransportError(error); - return complete({ - scenarioId: caseRecord.id, - suite: caseRecord.suite, - passed: false, - classification: classified.classification, - secondaryCode: classified.secondaryCode, - assertionResults: [], - diagnostics, - routeSubject: fallbackRouteSubject(routeContext), - transportError: transportCode, - }); + diagnostics.push(error instanceof Error ? `${error.name}:${error.message}` : String(error)); + let classified = classifyTransportError(error); + if (error instanceof LabSandboxError) classified = { classification: error.code === "harness_failure" ? "harness_failure" : "budget_exhausted", secondaryCode: error.code }; + if (error instanceof LabDestinationError) classified = { classification: error.code === "network_blocked" ? "network_failure" : "harness_failure", secondaryCode: error.code }; + return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }); } } - -function fallbackRouteSubject(routeContext: LabRouteContext): RouteSubjectV1 { - return { - subjectSchemaVersion: 1, - subjectKind: "route", - providerId: routeContext.providerId, - providerInstanceFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", - clientModelId: routeContext.clientModelId, - upstreamModelId: routeContext.upstreamModelId, - effectiveAdapter: routeContext.effectiveAdapter, - inboundProtocol: routeContext.inboundProtocol, - upstreamProtocol: routeContext.upstreamProtocol, - surface: routeContext.surface, - opencodexCompatibilityVersion: "live-v1", - behaviorFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", - endpointFingerprint: "0000000000000000000000000000000000000000000000000000000000000000", - dependencies: [], - }; -} diff --git a/src/lab/live/manifest.ts b/src/lab/live/manifest.ts index eda89fb55..c78adc45e 100644 --- a/src/lab/live/manifest.ts +++ b/src/lab/live/manifest.ts @@ -1,122 +1,83 @@ -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { fixtureDigest } from "../conformance/digest"; -import type { - CaseAuthority, - CaseRecord, - FailureClassification, - FailureRule, -} from "../conformance/types"; +import { fixtureDigest, jcsStringify } from "../digest"; +import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule } from "../conformance/types"; import { CL03_LIVE_SUITES, SYNTHETIC_MARKER } from "../conformance/types"; const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); const AUTHORITY_FILE = "024_live_v1_cases.json"; -const FIXTURES_DIR = join(MODULE_DIR, "..", "conformance", "fixtures"); +const RUNTIME_AUTHORITY = join(MODULE_DIR, "..", "conformance", "fixtures", "live-v1-cases.json"); +const REPO_AUTHORITY = join(MODULE_DIR, "..", "..", "..", "devlog", "_plan", "260807_compatibility_lab", AUTHORITY_FILE); + +function readAuthority(path: string): CaseAuthority { return JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; } + +/** During source-tree execution, fail closed if the packaged authority copy drifts from normative 024. */ +export function assertLiveAuthorityMirror(runtime: CaseAuthority): void { + if (!existsSync(REPO_AUTHORITY)) return; + const normative = readAuthority(REPO_AUTHORITY); + if (jcsStringify(runtime) !== jcsStringify(normative)) throw new Error("harness_failure: live authority runtime copy drift"); +} export function loadLiveCaseAuthority(): CaseAuthority { - const path = join(FIXTURES_DIR, "live-v1-cases.json"); - const raw = JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; + const raw = readAuthority(RUNTIME_AUTHORITY); validateLiveAuthority(raw); + assertLiveAuthorityMirror(raw); return raw; } -export function discoverLiveScenarios( - authority: CaseAuthority, - suites: readonly string[] = CL03_LIVE_SUITES, -): CaseRecord[] { - return authority.cases.filter((c) => suites.includes(c.suite)); +export function discoverLiveScenarios(authority: CaseAuthority, suites: readonly string[] = CL03_LIVE_SUITES): CaseRecord[] { + return authority.cases.filter((row) => suites.includes(row.suite)); } export function expandLiveScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { const defaults = authority.manifestDefaults; - const fixtures = caseRecord.initiatingRequest - ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] - : [fixtureRef(caseRecord.fixture, authority)]; + const fixtures = caseRecord.initiatingRequest ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] : [fixtureRef(caseRecord.fixture, authority)]; return { - schemaVersion: authority.schemaVersion, - id: caseRecord.id, - version: defaults.version, - suite: { - id: caseRecord.suite, - version: defaults.suiteVersion, - evidenceLayer: defaults.evidenceLayer, - }, - evidenceLayer: defaults.evidenceLayer, - capability: caseRecord.capability, - verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, - requirements: caseRecord.requirements, - fixtures, - executionLimits: defaults.executionLimits, - executionMode: defaults.executionMode, - assertions: caseRecord.assertions, + schemaVersion: authority.schemaVersion, id: caseRecord.id, version: defaults.version, + suite: { id: caseRecord.suite, version: defaults.suiteVersion, evidenceLayer: defaults.evidenceLayer }, + evidenceLayer: defaults.evidenceLayer, capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, requirements: caseRecord.requirements, + fixtures, executionLimits: defaults.executionLimits, executionMode: defaults.executionMode, assertions: caseRecord.assertions, ...(caseRecord.expectedFailure ? { expectedFailure: caseRecord.expectedFailure } : {}), - failureRules: expandLiveFailureRules(caseRecord, authority), - artifactPolicy: defaults.artifactPolicy, - freshness: defaults.freshness, + failureRules: expandLiveFailureRules(caseRecord, authority), artifactPolicy: defaults.artifactPolicy, freshness: defaults.freshness, }; } function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): Record { const bytes = new TextEncoder().encode(fixture.bytesUtf8); - return { - id: fixture.id, - role: fixture.role, - mediaType: fixture.mediaType, - digest: fixture.digest, - byteLength: bytes.byteLength, - syntheticMarker: SYNTHETIC_MARKER, - provenance: { - kind: "lab_authored", - authority: AUTHORITY_FILE, - sourceCommit: authority.sourceCommit, - }, - }; + return { id: fixture.id, role: fixture.role, mediaType: fixture.mediaType, digest: fixture.digest, byteLength: bytes.byteLength, + syntheticMarker: SYNTHETIC_MARKER, provenance: { kind: "lab_authored", authority: AUTHORITY_FILE, sourceCommit: authority.sourceCommit } }; } function expandLiveFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { const setName = authority.manifestDefaults.failureRuleSet; const ruleSet = authority.failureRuleSets[setName]; - if (!Array.isArray(ruleSet)) { - throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); - } + if (!Array.isArray(ruleSet)) throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); const base = [...ruleSet]; if (!caseRecord.expectedFailure) return base; const template = authority.expectedFailureRuleTemplate; - const controlRule: FailureRule = { - id: template.id, - match: [...template.match], - classification: caseRecord.expectedFailure.expectedClass as FailureClassification, - secondaryCode: caseRecord.expectedFailure.expectedCode, - verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", - retry: template.retry, - expected: template.expected, - }; - const idx = base.findIndex((r) => r.id === "required-assertion"); - if (idx >= 0) base.splice(idx, 0, controlRule); - else base.push(controlRule); + const controlRule: FailureRule = { id: template.id, match: [...template.match], classification: caseRecord.expectedFailure.expectedClass as FailureClassification, + secondaryCode: caseRecord.expectedFailure.expectedCode, verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", + retry: template.retry, expected: template.expected }; + const idx = base.findIndex((row) => row.id === "required-assertion"); + if (idx >= 0) base.splice(idx, 0, controlRule); else base.push(controlRule); return base; } function validateLiveAuthority(authority: CaseAuthority): void { - if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); - if (!authority.sourceCommit || typeof authority.sourceCommit !== "string") { - throw new Error("missing sourceCommit"); - } - if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); - if (authority.manifestDefaults.evidenceLayer !== "live_route_compatibility") { - throw new Error("invalid evidenceLayer for live authority"); - } + if (authority.schemaVersion !== 1 || authority.manifestDefaults.evidenceLayer !== "live_route_compatibility") throw new Error("invalid live authority"); + if (!authority.sourceCommit || !Array.isArray(authority.cases) || authority.cases.length !== 10) throw new Error("invalid live authority case set"); + const ids = new Set(); for (const caseRecord of authority.cases) { + if (ids.has(caseRecord.id)) throw new Error(`duplicate live scenario ${caseRecord.id}`); + ids.add(caseRecord.id); + if (!caseRecord.id.split(".").includes("live")) throw new Error(`non-live scenario in CL-03 authority: ${caseRecord.id}`); const bytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); - if (fixtureDigest(bytes) !== caseRecord.fixture.digest) { - throw new Error(`${caseRecord.id}: fixture digest mismatch`); - } + if (fixtureDigest(bytes) !== caseRecord.fixture.digest) throw new Error(`${caseRecord.id}: fixture digest mismatch`); if (caseRecord.initiatingRequest) { const initBytes = new TextEncoder().encode(caseRecord.initiatingRequest.bytesUtf8); - if (fixtureDigest(initBytes) !== caseRecord.initiatingRequest.digest) { - throw new Error(`${caseRecord.id}: initiatingRequest digest mismatch`); - } + if (fixtureDigest(initBytes) !== caseRecord.initiatingRequest.digest) throw new Error(`${caseRecord.id}: initiatingRequest digest mismatch`); } expandLiveScenario(caseRecord, authority); } diff --git a/src/lab/live/runner.ts b/src/lab/live/runner.ts index 578c962fe..ed4e2ae99 100644 --- a/src/lab/live/runner.ts +++ b/src/lab/live/runner.ts @@ -1,32 +1,19 @@ +import { CL03_LIVE_SUITES } from "../conformance/types"; +import { isLiveCaseApplicableToRoute, runLiveScenario, type LiveExecutorOptions } from "./executor"; import { discoverLiveScenarios, loadLiveCaseAuthority } from "./manifest"; -import { runLiveScenario, type LiveExecutorOptions } from "./executor"; import type { LabRouteContext, LiveScenarioRunResult } from "./types"; -import { CL03_LIVE_SUITES } from "../conformance/types"; -export interface LiveRunSummary { - total: number; - passed: number; - failed: number; - results: LiveScenarioRunResult[]; -} +export interface LiveRunSummary { total: number; passed: number; failed: number; results: LiveScenarioRunResult[] } -export async function runLiveSuite( - routeContext: LabRouteContext, - suites: readonly string[] = CL03_LIVE_SUITES, - opts: LiveExecutorOptions = {}, -): Promise { +export async function runLiveSuite(routeContext: LabRouteContext, suites: readonly string[] = CL03_LIVE_SUITES, opts: LiveExecutorOptions = {}): Promise { const authority = loadLiveCaseAuthority(); - const scenarios = discoverLiveScenarios(authority, suites); - if (scenarios.length === 0) throw new Error("harness_failure: no CL-03 live scenarios discovered"); + const scenarios = discoverLiveScenarios(authority, suites).filter((scenario) => isLiveCaseApplicableToRoute(scenario, routeContext)); const results: LiveScenarioRunResult[] = []; - for (const scenario of scenarios) { - results.push(await runLiveScenario(scenario, routeContext, opts)); - } - const passed = results.filter((r) => r.passed).length; + for (const scenario of scenarios) results.push(await runLiveScenario(scenario, routeContext, opts)); + const passed = results.filter((row) => row.passed).length; return { total: results.length, passed, failed: results.length - passed, results }; } export function listLiveScenarioIds(suites: readonly string[] = CL03_LIVE_SUITES): string[] { - const authority = loadLiveCaseAuthority(); - return discoverLiveScenarios(authority, suites).map((s) => s.id); + return discoverLiveScenarios(loadLiveCaseAuthority(), suites).map((scenario) => scenario.id); } diff --git a/src/lab/live/sandbox.ts b/src/lab/live/sandbox.ts index 5116f08d5..b3b87abc6 100644 --- a/src/lab/live/sandbox.ts +++ b/src/lab/live/sandbox.ts @@ -1,107 +1,50 @@ import type { LiveRunConfig } from "./types"; -const PROXY_ENV_VARS = [ - "HTTP_PROXY", - "HTTPS_PROXY", - "ALL_PROXY", - "NO_PROXY", - "http_proxy", - "https_proxy", - "all_proxy", - "no_proxy", -] as const; - -const ALLOWED_ENV = { - TZ: "UTC", - NO_COLOR: "1", -} as const; +const PROXY_ENV_VARS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const; +const ALLOWED_ENV = { TZ: "UTC", NO_COLOR: "1" } as const; export class LabSandboxError extends Error { override readonly name = "LabSandboxError"; - constructor( - message: string, - readonly code: string, - ) { - super(message); - } + constructor(message: string, readonly code: string) { super(message); } } -/** Reject inherited proxy environment variables for Lab live runs. */ export function rejectProxyEnvironment(env: NodeJS.ProcessEnv = process.env): void { for (const name of PROXY_ENV_VARS) { const value = env[name]; - if (value !== undefined && value !== "") { - throw new LabSandboxError(`proxy environment variable ${name} is forbidden`, "harness_failure"); - } + if (value !== undefined && value !== "") throw new LabSandboxError(`proxy environment variable ${name} is forbidden`, "harness_failure"); } } -/** Build the constructed non-inherited environment view for live runs. */ -export function labSandboxEnvironment(): Readonly> { - return Object.freeze({ ...ALLOWED_ENV }); -} +export function labSandboxEnvironment(): Readonly> { return Object.freeze({ ...ALLOWED_ENV }); } export interface SandboxResourceState { - requests: number; - inputBytes: number; - outputBytes: number; - toolCalls: number; - artifacts: number; - artifactBytes: number; - childProcesses: number; + requests: number; inputBytes: number; outputBytes: number; outputTokens: number; toolCalls: number; + artifacts: number; artifactBytes: number; childProcesses: number; peakMemoryBytes: number; } export function createSandboxResourceState(): SandboxResourceState { - return { - requests: 0, - inputBytes: 0, - outputBytes: 0, - toolCalls: 0, - artifacts: 0, - artifactBytes: 0, - childProcesses: 0, - }; + return { requests: 0, inputBytes: 0, outputBytes: 0, outputTokens: 0, toolCalls: 0, artifacts: 0, artifactBytes: 0, childProcesses: 0, peakMemoryBytes: process.memoryUsage().rss }; } -export function enforceSandboxLimits( - state: SandboxResourceState, - limits: LiveRunConfig, - delta: Partial = {}, -): void { - const next = { - requests: state.requests + (delta.requests ?? 0), - inputBytes: state.inputBytes + (delta.inputBytes ?? 0), - outputBytes: state.outputBytes + (delta.outputBytes ?? 0), - toolCalls: state.toolCalls + (delta.toolCalls ?? 0), - artifacts: state.artifacts + (delta.artifacts ?? 0), - artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), - childProcesses: state.childProcesses + (delta.childProcesses ?? 0), +export function enforceSandboxLimits(state: SandboxResourceState, limits: LiveRunConfig, delta: Partial = {}): void { + const next: SandboxResourceState = { + requests: state.requests + (delta.requests ?? 0), inputBytes: state.inputBytes + (delta.inputBytes ?? 0), + outputBytes: state.outputBytes + (delta.outputBytes ?? 0), outputTokens: state.outputTokens + (delta.outputTokens ?? 0), + toolCalls: state.toolCalls + (delta.toolCalls ?? 0), artifacts: state.artifacts + (delta.artifacts ?? 0), + artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), childProcesses: state.childProcesses + (delta.childProcesses ?? 0), + peakMemoryBytes: Math.max(state.peakMemoryBytes, delta.peakMemoryBytes ?? process.memoryUsage().rss), }; - if (next.childProcesses > limits.maxChildProcesses) { - throw new LabSandboxError("child process limit exceeded", "harness_failure"); - } - if (next.requests > limits.maxRequests) { - throw new LabSandboxError("request limit exceeded", "request_limit"); - } - if (next.inputBytes > limits.maxInputBytes) { - throw new LabSandboxError("input byte limit exceeded", "request_limit"); - } - if (next.outputBytes > limits.maxOutputBytes) { - throw new LabSandboxError("output byte limit exceeded", "request_limit"); - } - if (next.toolCalls > limits.maxToolCalls) { - throw new LabSandboxError("tool call limit exceeded", "request_limit"); - } - if (next.artifacts > limits.maxArtifacts) { - throw new LabSandboxError("artifact limit exceeded", "request_limit"); - } - if (next.artifactBytes > limits.aggregateArtifactBytes) { - throw new LabSandboxError("aggregate artifact limit exceeded", "request_limit"); - } + if (next.childProcesses > limits.maxChildProcesses) throw new LabSandboxError("child process limit exceeded", "child_process_limit"); + if (next.requests > limits.maxRequests) throw new LabSandboxError("request limit exceeded", "request_limit"); + if (next.inputBytes > limits.maxInputBytes) throw new LabSandboxError("input byte limit exceeded", "input_byte_limit"); + if (next.outputBytes > limits.maxOutputBytes) throw new LabSandboxError("output byte limit exceeded", "output_byte_limit"); + if (next.outputTokens > limits.maxOutputTokens) throw new LabSandboxError("output token limit exceeded", "output_token_limit"); + if (next.toolCalls > limits.maxToolCalls) throw new LabSandboxError("tool call limit exceeded", "tool_call_limit"); + if (next.artifacts > limits.maxArtifacts) throw new LabSandboxError("artifact limit exceeded", "artifact_byte_limit"); + if (next.artifactBytes > limits.aggregateArtifactBytes) throw new LabSandboxError("aggregate artifact limit exceeded", "artifact_byte_limit"); Object.assign(state, next); } -/** Prepare sandbox: reject proxy env and return allowed environment. */ export function prepareLiveSandbox(env: NodeJS.ProcessEnv = process.env): Readonly> { rejectProxyEnvironment(env); return labSandboxEnvironment(); diff --git a/src/lab/live/suite-manifest.ts b/src/lab/live/suite-manifest.ts index efa88f53f..203bbe650 100644 --- a/src/lab/live/suite-manifest.ts +++ b/src/lab/live/suite-manifest.ts @@ -2,66 +2,40 @@ import { scenarioManifestDigest, suiteManifestDigest } from "../digest"; import type { CaseAuthority, CaseRecord, VerificationRole } from "../conformance/types"; import { expandLiveScenario } from "./manifest"; -export interface LiveSuiteScenarioRefV1 { - id: string; - version: string; - role: VerificationRole; - manifestDigest: string; +export interface LiveSuiteScenarioRefV1 { id: string; version: string; role: VerificationRole; manifestDigest: string } +export interface LiveSuiteManifestV1 { + schemaVersion: 1; id: string; version: string; evidenceLayer: string; capability: string; + assertionDslVersion: string; evidenceSchemaVersion: string; freshness: { maxAgeMs: number | null }; + contradictionRule: string; scenarios: LiveSuiteScenarioRefV1[]; verificationRule: string; } -export interface LiveSuiteManifestV1 { - schemaVersion: 1; - id: string; - version: string; - evidenceLayer: string; - capability: string; - assertionDslVersion: string; - evidenceSchemaVersion: string; - freshness: { maxAgeMs: number | null }; - contradictionRule: string; - scenarios: LiveSuiteScenarioRefV1[]; - verificationRule: string; +function casesForSuiteManifest(suiteId: string, authority: CaseAuthority): { own: CaseRecord[]; referenced: CaseRecord[] } { + const own = authority.cases.filter((row) => row.suite === suiteId); + if (own.length === 0) throw new Error(`unknown live suite ${suiteId}`); + if (suiteId !== "codex-core") return { own, referenced: own }; + const prerequisite = authority.cases.find((row) => row.id === "responses-core.live.basic-turn"); + if (!prerequisite) throw new Error("harness_failure: codex live prerequisite missing"); + return { own, referenced: [...own, prerequisite] }; } -export function expandLiveSuiteManifest( - suiteId: string, - authority: CaseAuthority, -): LiveSuiteManifestV1 { - const cases = authority.cases.filter((c) => c.suite === suiteId); - if (cases.length === 0) throw new Error(`unknown live suite ${suiteId}`); +export function expandLiveSuiteManifest(suiteId: string, authority: CaseAuthority): LiveSuiteManifestV1 { + const { own, referenced } = casesForSuiteManifest(suiteId, authority); const defaults = authority.manifestDefaults; - const capability = cases[0]!.capability; - const scenarios: LiveSuiteScenarioRefV1[] = cases - .map((caseRecord) => ({ - id: caseRecord.id, - version: String(defaults.version), - role: caseRecord.verificationRole ?? defaults.verificationRole, - manifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), - })) - .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); - + const scenarios = referenced.map((caseRecord) => ({ + id: caseRecord.id, version: String(defaults.version), role: caseRecord.verificationRole ?? defaults.verificationRole, + manifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), + })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0); return { - schemaVersion: 1, - id: suiteId, - version: String(defaults.suiteVersion), - evidenceLayer: defaults.evidenceLayer, - capability, - assertionDslVersion: authority.assertionDslVersion, - evidenceSchemaVersion: authority.evidenceSchemaVersion, - freshness: defaults.freshness ?? { maxAgeMs: null }, - contradictionRule: "newest-required-observation-v1", - scenarios, + schemaVersion: 1, id: suiteId, version: String(defaults.suiteVersion), evidenceLayer: defaults.evidenceLayer, + capability: own[0]!.capability, assertionDslVersion: authority.assertionDslVersion, evidenceSchemaVersion: authority.evidenceSchemaVersion, + freshness: defaults.freshness ?? { maxAgeMs: null }, contradictionRule: "newest-required-observation-v1", scenarios, verificationRule: "all-applicable-required-pass-v1", }; } -export function liveSuiteManifestObjectForCase( - caseRecord: CaseRecord, - authority: CaseAuthority, -): Record { +export function liveSuiteManifestObjectForCase(caseRecord: CaseRecord, authority: CaseAuthority): Record { return expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record; } - export function liveSuiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string { return suiteManifestDigest(expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record); } diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index 4d5528d18..d32811e28 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -1,12 +1,7 @@ -import { providerRedirectError } from "../../lib/provider-outbound"; -import { assertLeaseScope } from "./credential-lease"; +import { assertLeaseScope, LabCredentialError } from "./credential-lease"; import type { - LabCredentialLeaseV1, - LabDestinationV1, - LabTransport, - TransportErrorCode, - TransportRequest, - TransportResponse, + LabCredentialLeaseV1, LabDestinationV1, LabPinnedSender, LabTransport, LabTransportRequest, + LabTransportResponse, LiveRunConfig, TransportErrorCode, } from "./types"; export interface MockTransportEntry { @@ -16,38 +11,23 @@ export interface MockTransportEntry { body: string; error?: TransportErrorCode; } +export interface MockTransportOptions { entries?: MockTransportEntry[]; onRequest?: (req: LabTransportRequest) => void } -export interface MockTransportOptions { - entries?: MockTransportEntry[]; - onRequest?: (req: TransportRequest) => void; -} - -/** Injectable mock transport for tests — returns fixture upstream responses. */ +/** Test seam only. No live runner path creates this implicitly. */ export function createMockTransport(opts: MockTransportOptions = {}): LabTransport { let callIndex = 0; return { - async request(req: TransportRequest): Promise { + async request(req): Promise { opts.onRequest?.(req); const entry = opts.entries?.[callIndex] ?? opts.entries?.[opts.entries.length - 1]; callIndex += 1; - if (!entry) { - throw transportError("harness_failure", "no mock transport entry"); - } - if (entry.error) throw transportError(entry.error, entry.error); + if (!entry) throw new TransportError("harness_failure", "no mock transport entry"); + if (entry.error) throw new TransportError(entry.error, entry.error); if (entry.matchPath) { - const matches = typeof entry.matchPath === "string" - ? req.path.includes(entry.matchPath) - : entry.matchPath.test(req.path); - if (!matches) throw transportError("harness_failure", "mock path mismatch"); - } - if (entry.status >= 300 && entry.status < 400) { - throw transportError("redirect_blocked", "redirect response"); + const ok = typeof entry.matchPath === "string" ? req.path.includes(entry.matchPath) : entry.matchPath.test(req.path); + if (!ok) throw new TransportError("harness_failure", "mock path mismatch"); } - return { - status: entry.status, - headers: { ...(entry.headers ?? {}), "content-type": entry.headers?.["content-type"] ?? "application/json" }, - body: entry.body, - }; + return { status: entry.status, headers: { "content-type": "application/json", ...(entry.headers ?? {}) }, body: entry.body }; }, }; } @@ -55,90 +35,80 @@ export function createMockTransport(opts: MockTransportOptions = {}): LabTranspo export interface PinnedTransportOptions { destination: LabDestinationV1; lease: LabCredentialLeaseV1; - fetch?: typeof globalThis.fetch; + sender: LabPinnedSender; + limits: LiveRunConfig; transportId?: string; } -/** Production path: provider-outbound with pinned destination; lease consumed per request. */ +function assertRelativeRequestPath(path: string): void { + if (!path.startsWith("/") || path.startsWith("//") || path.includes("://") || /[\r\n]/.test(path)) { + throw new TransportError("harness_failure", "request path attempted to widen destination"); + } +} + +function selectedPinnedAddress(destination: LabDestinationV1): { address: string; family: 4 | 6 } { + const row = destination.addresses.find((a) => a.family === 4) ?? destination.addresses[0]; + if (!row) throw new TransportError("harness_failure", "approved destination has no address"); + return { address: row.address, family: row.family }; +} + +/** Uses only the frozen approved address set. Secret injection is owned by the trusted sender, outside Lab. */ export function createPinnedTransport(opts: PinnedTransportOptions): LabTransport { const transportId = opts.transportId ?? "default"; return { - async request(req: TransportRequest): Promise { + async request(req): Promise { + assertRelativeRequestPath(req.path); assertLeaseScope(opts.lease, opts.destination, transportId); + const inputBytes = new TextEncoder().encode(req.body ?? "").byteLength; + if (inputBytes > opts.limits.maxInputBytes) throw new TransportError("input_byte_limit", "request exceeds input byte budget"); opts.lease.consume(); - const url = `${opts.destination.scheme}://${opts.destination.host}:${opts.destination.port}${opts.destination.basePath}${req.path}`; - const fetchFn = opts.fetch ?? globalThis.fetch; - const response = await fetchFn(url, { - method: req.method, - headers: req.headers, - body: req.body, - redirect: "manual", - }); - const redirectError = await providerRedirectError(response, url); - if (redirectError) throw transportError("redirect_blocked", redirectError); - if (response.status === 401 || response.status === 403) { - throw transportError("auth_blocked", `HTTP ${response.status}`); - } - if (response.status === 429) { - throw transportError("quota_blocked", `HTTP ${response.status}`); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live request total timeout")), opts.limits.totalTimeoutMs); + const onAbort = () => controller.abort(req.signal?.reason); + req.signal?.addEventListener("abort", onAbort, { once: true }); + try { + const response = await opts.sender(opts.lease, opts.destination, selectedPinnedAddress(opts.destination), req, controller.signal, opts.limits); + const outputBytes = new TextEncoder().encode(response.body).byteLength; + if (outputBytes > opts.limits.maxOutputBytes) throw new TransportError("output_byte_limit", "response exceeds output byte budget"); + if (response.status >= 300 && response.status < 400) throw new TransportError("redirect_blocked", "redirect response"); + if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`); + if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429"); + if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); + if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`); + return response; + } catch (error) { + if (error instanceof LabCredentialError) { + throw new TransportError(error.code === "budget_exhausted" ? "request_limit" : "harness_failure", error.message); + } + throw error; + } finally { + clearTimeout(timer); + req.signal?.removeEventListener("abort", onAbort); } - if (response.status >= 500) { - throw transportError("provider_transient", `HTTP ${response.status}`); - } - const body = await response.text(); - const headers: Record = {}; - response.headers.forEach((value, key) => { - headers[key.toLowerCase()] = value; - }); - return { status: response.status, headers, body }; }, }; } export class TransportError extends Error { override readonly name = "TransportError"; - constructor( - readonly code: TransportErrorCode, - message: string, - ) { - super(message); - } -} - -function transportError(code: TransportErrorCode, message: string): TransportError { - return new TransportError(code, message); + constructor(readonly code: TransportErrorCode, message: string) { super(message); } } export function classifyTransportError(error: unknown): { classification: string; secondaryCode: string } { - if (error instanceof TransportError) { - switch (error.code) { - case "auth_blocked": - return { classification: "authentication_blocked", secondaryCode: "auth_blocked" }; - case "quota_blocked": - return { classification: "quota_blocked", secondaryCode: "quota_blocked" }; - case "network_blocked": - return { classification: "network_failure", secondaryCode: "network_blocked" }; - case "region_blocked": - return { classification: "region_blocked", secondaryCode: "region_blocked" }; - case "provider_transient": - return { classification: "provider_transient", secondaryCode: "provider_transient" }; - case "connect_timeout": - case "first_byte_timeout": - case "inactivity_timeout": - case "total_timeout": - return { classification: "timeout", secondaryCode: error.code }; - case "request_limit": - return { classification: "budget_exhausted", secondaryCode: "scenario_resource_limit" }; - case "redirect_blocked": - case "destination_mismatch": - case "host_sni_mismatch": - case "harness_failure": - return { classification: "harness_failure", secondaryCode: error.code }; - default: { - const _never: never = error.code; - return { classification: "harness_failure", secondaryCode: String(_never) }; - } - } + const code = error instanceof TransportError ? error.code : undefined; + switch (code) { + case "auth_blocked": return { classification: "authentication_blocked", secondaryCode: code }; + case "quota_blocked": return { classification: "quota_blocked", secondaryCode: code }; + case "network_blocked": return { classification: "network_failure", secondaryCode: code }; + case "region_blocked": return { classification: "region_blocked", secondaryCode: code }; + case "provider_transient": return { classification: "provider_transient", secondaryCode: code }; + case "connect_timeout": case "first_byte_timeout": case "inactivity_timeout": case "total_timeout": + return { classification: "timeout", secondaryCode: code }; + case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit": + case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit": + return { classification: "budget_exhausted", secondaryCode: code }; + case "redirect_blocked": case "destination_mismatch": case "host_sni_mismatch": case "harness_failure": + return { classification: "harness_failure", secondaryCode: code }; + default: return { classification: "harness_failure", secondaryCode: "execution_error" }; } - return { classification: "harness_failure", secondaryCode: "execution_error" }; } diff --git a/src/lab/live/types.ts b/src/lab/live/types.ts index 16724ded5..693f47db7 100644 --- a/src/lab/live/types.ts +++ b/src/lab/live/types.ts @@ -1,4 +1,5 @@ -import type { RouteDependencyV1 } from "../events/types"; +import type { NormalizedObservation } from "../conformance/types"; +import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types"; export interface LabDestinationV1 { readonly scheme: "http" | "https"; @@ -11,8 +12,28 @@ export interface LabDestinationV1 { readonly fingerprint: string; } +export type LabBehaviorSource = + | "request" + | "model_override" + | "provider_config" + | "registry_runtime_default" + | "generated_model_metadata" + | "global_config" + | "adapter_default" + | "lab_forced"; + +export interface LabBehaviorValue { + source: LabBehaviorSource; + value: unknown; +} + +/** Authoritative effective values emitted by the production route/model/adapter resolver. */ +export type LabBehaviorValues = Record; + export interface LabRouteContext { providerId: string; + /** Config-owner identity only; HMACed before it enters evidence. */ + providerInstanceKey: string; clientModelId: string; upstreamModelId: string; effectiveAdapter: string; @@ -20,9 +41,14 @@ export interface LabRouteContext { upstreamProtocol: string; surface: string; baseUrl: string; + /** Generated compatibility hash, not package marketing version. */ + opencodexCompatibilityVersion: string; + /** Closed, effective behavior inputs from the production resolver. */ + behaviorValues: LabBehaviorValues; allowPrivateNetwork?: boolean; labRunApproval?: boolean; requiredClaims?: string[]; + availableHarnessFeatures?: string[]; dependencies?: RouteDependencyV1[]; } @@ -59,35 +85,74 @@ export type TransportErrorCode = | "inactivity_timeout" | "total_timeout" | "request_limit" + | "input_byte_limit" + | "output_byte_limit" + | "output_token_limit" + | "tool_call_limit" + | "artifact_byte_limit" + | "memory_limit" + | "child_process_limit" | "harness_failure"; -export interface TransportRequest { +export interface LabTransportRequest { method: "GET" | "POST"; path: string; headers?: Record; body?: string; + signal?: AbortSignal; } -export interface TransportResponse { +export interface LabTransportResponse { status: number; headers: Record; body: string; } +export type TransportRequest = LabTransportRequest; +export type TransportResponse = LabTransportResponse; + export interface LabTransport { - request(req: TransportRequest): Promise; + request(req: LabTransportRequest): Promise; } export const LAB_CREDENTIAL_LEASE = Symbol("LabCredentialLeaseV1"); +/** Opaque capability. Scope metadata and secret material are deliberately not public fields. */ export interface LabCredentialLeaseV1 { readonly [LAB_CREDENTIAL_LEASE]: true; - readonly destinationFingerprint: string; - readonly transportId: string; readonly remainingRequests: number; consume(): void; } +export interface LabPinnedAddress { + address: string; + family: 4 | 6; +} + +/** Trusted provider/credential transport seam. Implementations own secret injection outside Lab. */ +export type LabPinnedSender = ( + lease: LabCredentialLeaseV1, + destination: LabDestinationV1, + pinned: LabPinnedAddress, + request: LabTransportRequest, + signal: AbortSignal, + limits: LiveRunConfig, +) => Promise; + +export interface LabRouteExecutorInput { + routeContext: LabRouteContext; + destination: LabDestinationV1; + routeSubject: RouteSubjectV1; + scenarioId: string; + initiatingRequest?: string; + limits: LiveRunConfig; + signal: AbortSignal; + environment: Readonly>; +} + +/** Trusted exact-route execution seam. It must use existing route/adapters and return client-observed normalization. */ +export type LabRouteExecutor = (input: LabRouteExecutorInput) => Promise; + export interface LiveScenarioRunResult { scenarioId: string; suite: string; @@ -98,6 +163,6 @@ export interface LiveScenarioRunResult { secondaryCode?: string; assertionResults: import("../conformance/types").AssertionResult[]; diagnostics: string[]; - routeSubject: import("../events/types").RouteSubjectV1; + routeSubject?: RouteSubjectV1; transportError?: TransportErrorCode; } diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index 5a9215770..0d5d190b2 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -1,14 +1,6 @@ -/** - * Persistence seam: transform CL-03 live probe results into valid observation events. - */ +// CL-03 live result -> CL-02 immutable observation persistence. import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; -import { - LAB_EVENT_SCHEMA_VERSION, - LAB_PRODUCER, - LAB_PRODUCER_VERSION, - OBSERVATION_LIMIT_NAMES, - type ObservationOutcome, -} from "../constants"; +import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PRODUCER_VERSION, OBSERVATION_LIMIT_NAMES, type ObservationOutcome } from "../constants"; import { fixtureDigest, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; import type { ObservationEvent } from "../events/types"; import { assignEventId } from "../events/validate"; @@ -19,220 +11,59 @@ import { expandLiveScenario } from "../live/manifest"; import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; import type { LiveScenarioRunResult } from "../live/types"; -export interface PersistLiveOptions { - configDir?: string; - recordedAt?: number; - startedAt?: number; - completedAt?: number; - producerVersion?: string; - artifactStore?: ArtifactStore; -} - -export interface PersistedLiveObservation { - event: ObservationEvent; - ledgerPath: string; -} +export interface PersistLiveOptions { configDir?: string; recordedAt?: number; startedAt?: number; completedAt?: number; producerVersion?: string; artifactStore?: ArtifactStore } +export interface PersistedLiveObservation { event: ObservationEvent; ledgerPath: string } function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; - switch (result.classification) { - case "timeout": - case "budget_exhausted": - case "authentication_blocked": - case "quota_blocked": - case "region_blocked": - case "network_failure": - case "provider_transient": - return "blocked"; - case "inconclusive": - case "harness_failure": - return "inconclusive"; - case "protocol_failure": - case "capability_failure": - case "behavioral_failure": - return "fail"; - default: - return "inconclusive"; - } + if (["timeout", "budget_exhausted", "authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient"].includes(result.classification)) return "blocked"; + if (["inconclusive", "harness_failure"].includes(result.classification)) return "inconclusive"; + if (["protocol_failure", "capability_failure", "behavioral_failure"].includes(result.classification)) return "fail"; + return "inconclusive"; } -function requireExecutionTimes( - result: LiveScenarioRunResult, - opts: PersistLiveOptions, -): { startedAt: number; completedAt: number } { - const startedAt = opts.startedAt ?? result.startedAt; - const completedAt = opts.completedAt ?? result.completedAt; - if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt)) { - throw new Error("real startedAt/completedAt are required for persisted live evidence"); - } - if (startedAt < 0 || completedAt < startedAt) { - throw new Error("invalid persisted live execution timestamps"); - } +function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveOptions): { startedAt: number; completedAt: number } { + const startedAt = opts.startedAt ?? result.startedAt; const completedAt = opts.completedAt ?? result.completedAt; + if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt) || startedAt < 0 || completedAt < startedAt) throw new Error("invalid persisted live execution timestamps"); return { startedAt, completedAt }; } -export function observationFromLiveResult( - result: LiveScenarioRunResult, - caseRecord: CaseRecord, - authority: CaseAuthority, - opts: PersistLiveOptions = {}, -): { event: ObservationEvent; artifacts: ReturnType[] } { - const paths = ensureLabDirs(opts.configDir); - const ownsStore = !opts.artifactStore; - const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); +export function observationFromLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): { event: ObservationEvent; artifacts: ReturnType[] } { + if (!result.routeSubject) throw new Error("live evidence without an exact RouteSubjectV1 is not persistable"); + const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const { startedAt, completedAt } = requireExecutionTimes(result, opts); - const recordedAt = opts.recordedAt ?? completedAt; - - const expandedScenario = expandLiveScenario(caseRecord, authority); - const scenarioDigest = scenarioManifestDigest(expandedScenario); - const suiteExpanded = liveSuiteManifestObjectForCase(caseRecord, authority); - const suiteDigest = suiteManifestDigest(suiteExpanded); - - const fixtureDigests: string[] = []; - const artifacts: ReturnType[] = []; - + const { startedAt, completedAt } = requireExecutionTimes(result, opts); const recordedAt = opts.recordedAt ?? completedAt; + const expandedScenario = expandLiveScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); + const suiteExpanded = liveSuiteManifestObjectForCase(caseRecord, authority); const suiteDigest = suiteManifestDigest(suiteExpanded); + const fixtureDigests: string[] = []; const artifacts: ReturnType[] = []; const putFixture = (fixture: CaseRecord["fixture"]) => { - const bytes = new TextEncoder().encode(fixture.bytesUtf8); - const digest = fixtureDigest(bytes); - fixtureDigests.push(digest); - artifacts.push( - store.put({ - artifactClass: "fixture", - payload: bytes, - expectedDigest: digest, - mediaType: fixture.mediaType, - }), - ); + const bytes = new TextEncoder().encode(fixture.bytesUtf8); const digest = fixtureDigest(bytes); fixtureDigests.push(digest); + artifacts.push(store.put({ artifactClass: "fixture", payload: bytes, expectedDigest: digest, mediaType: fixture.mediaType })); }; - putFixture(caseRecord.fixture); - if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); - - artifacts.push( - store.put({ - artifactClass: "scenario_manifest", - payload: expandedScenario, - expectedDigest: scenarioDigest, - }), - ); - artifacts.push( - store.put({ - artifactClass: "suite_manifest", - payload: suiteExpanded, - expectedDigest: suiteDigest, - }), - ); - - const assertionReport = store.put({ - artifactClass: "assertion_report", - payload: { - scenarioId: result.scenarioId, - passed: result.passed, - classification: result.classification, - assertions: result.assertionResults.map((a) => ({ - id: a.id, - operator: a.operator, - required: a.required, - passed: a.passed, - observedSummary: a.observedSummary, - reason: a.reason, - })), - }, - }); - artifacts.push(assertionReport); - - const subject = result.routeSubject; - const subjectId = subjectIdForSubject(subject); - const outcome = outcomeFromLiveResult(result); - const authorityLimits = authority.manifestDefaults.executionLimits; - const limits: Record = {}; - for (const key of OBSERVATION_LIMIT_NAMES) { - if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; - } - + putFixture(caseRecord.fixture); if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); + artifacts.push(store.put({ artifactClass: "scenario_manifest", payload: expandedScenario, expectedDigest: scenarioDigest })); + artifacts.push(store.put({ artifactClass: "suite_manifest", payload: suiteExpanded, expectedDigest: suiteDigest })); + artifacts.push(store.put({ artifactClass: "assertion_report", payload: { scenarioId: result.scenarioId, passed: result.passed, classification: result.classification, + assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, observedSummary: row.observedSummary, reason: row.reason })) } })); + const subject = result.routeSubject; const subjectId = subjectIdForSubject(subject); const authorityLimits = authority.manifestDefaults.executionLimits; + const limits: Record = {}; for (const key of OBSERVATION_LIMIT_NAMES) if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; const eventWithoutId = { - schemaVersion: LAB_EVENT_SCHEMA_VERSION, - eventKind: "observation" as const, - recordedAt, - producer: LAB_PRODUCER, - producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, - evidenceLayer: "live_route_compatibility" as const, - scenarioId: caseRecord.id, - scenarioVersion: String(authority.manifestDefaults.version), - scenarioManifestDigest: scenarioDigest, - suiteId: caseRecord.suite, - suiteVersion: String(authority.manifestDefaults.suiteVersion), - suiteManifestDigest: suiteDigest, - fixtureDigests, - subject, - subjectId, - startedAt, - completedAt, - executionMode: "live" as const, - attempt: 1, - limits: { ...limits }, - outcome, - assertions: result.assertionResults.map((a) => ({ - id: a.id, - operator: a.operator, - required: a.required, - passed: a.passed, - expectedSummary: "see_assertion_report", - observedSummary: a.observedSummary.slice(0, 512), - ...(a.reason ? { reason: a.reason } : {}), - })), - ...(caseRecord.expectedFailure - ? { expectedFailure: { ...caseRecord.expectedFailure } } - : {}), - environment: { - runtime: { - platform: process.platform, - arch: process.arch, - bunVersion: process.versions.bun ?? Bun.version, - }, - }, - artifactRefs: artifacts, - ...(result.passed - ? {} - : { - failure: { - class: result.classification, - code: result.secondaryCode ?? result.classification, - retryable: false, - attribution: - result.classification === "harness_failure" - ? ("harness" as const) - : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) - ? ("environment" as const) - : ("route" as const), - }, - }), + schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, + evidenceLayer: "live_route_compatibility" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), scenarioManifestDigest: scenarioDigest, + suiteId: caseRecord.suite, suiteVersion: String(authority.manifestDefaults.suiteVersion), suiteManifestDigest: suiteDigest, fixtureDigests, subject, subjectId, + startedAt, completedAt, executionMode: "live" as const, attempt: 1, limits, outcome: outcomeFromLiveResult(result), + assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, expectedSummary: "see_assertion_report", observedSummary: row.observedSummary.slice(0, 512), ...(row.reason ? { reason: row.reason } : {}) })), + ...(caseRecord.expectedFailure ? { expectedFailure: { ...caseRecord.expectedFailure } } : {}), + environment: { runtime: { platform: process.platform, arch: process.arch, bunVersion: process.versions.bun ?? Bun.version } }, artifactRefs: artifacts, + ...(result.passed ? {} : { failure: { class: result.classification, code: result.secondaryCode ?? result.classification, retryable: false, + attribution: result.classification === "harness_failure" ? "harness" as const : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) ? "environment" as const : "route" as const } }), }; - - const event = assignEventId(eventWithoutId) as ObservationEvent; - return { event, artifacts }; - } finally { - if (ownsStore) store.close(); - } + return { event: assignEventId(eventWithoutId) as ObservationEvent, artifacts }; + } finally { if (ownsStore) store.close(); } } -export function persistLiveResult( - result: LiveScenarioRunResult, - caseRecord: CaseRecord, - authority: CaseAuthority, - opts: PersistLiveOptions = {}, -): PersistedLiveObservation { - const paths = ensureLabDirs(opts.configDir); - const ownsStore = !opts.artifactStore; - const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); - try { - const { event } = observationFromLiveResult(result, caseRecord, authority, { - ...opts, - artifactStore: store, - }); - appendLabEvent(paths.ledgerPath, event); - return { event, ledgerPath: paths.ledgerPath }; - } finally { - if (ownsStore) store.close(); - } +export function persistLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): PersistedLiveObservation { + const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { const { event } = observationFromLiveResult(result, caseRecord, authority, { ...opts, artifactStore: store }); appendLabEvent(paths.ledgerPath, event); return { event, ledgerPath: paths.ledgerPath }; } + finally { if (ownsStore) store.close(); } } diff --git a/src/lab/subject/behavior-fingerprint.ts b/src/lab/subject/behavior-fingerprint.ts index 9276d12ac..4cdc571aa 100644 --- a/src/lab/subject/behavior-fingerprint.ts +++ b/src/lab/subject/behavior-fingerprint.ts @@ -1,33 +1,77 @@ import { createHash } from "node:crypto"; import { jcsStringify } from "../digest"; -import type { LabRouteContext } from "../live/types"; +import type { LabBehaviorSource, LabBehaviorValues } from "../live/types"; -const CLOSED_BEHAVIOR_KEYS = [ - "wire.adapter", - "wire.inboundProtocol", - "wire.upstreamProtocol", - "wire.surface", - "route.allowPrivateNetwork", - "route.labRunApproval", +const CLOSED_KEYS = new Set([ + "wire.adapter", "wire.upstreamProtocol", "wire.responsesPath", "wire.commandCodeVersion", "wire.modelSuffixMode", + "auth.mode", "auth.transport", + "responses.stateful", "responses.upstreamStreaming", "responses.serviceTier", "responses.snapshotRepair", "responses.itemIdRepair", + "limits.contextWindow", "limits.maxInputTokens", "limits.maxOutputTokens", + "modalities.input", + "sampling.omitTemperature", "sampling.omitTopP", "sampling.omitPenalties", + "reasoning.supported", "reasoning.efforts", "reasoning.defaultEffort", "reasoning.effortMap", "reasoning.wireFormat", + "reasoning.summaryMode", "reasoning.replayMode", "reasoning.splitMode", "reasoning.toggleMode", "reasoning.budgetMode", + "tools.choiceRestrictions", "tools.parallel", "tools.hostedPreference", "tools.customFreeform", "tools.builtinNameEscaping", + "cache.forwarding", "cache.retention", "anthropic.eofPolicy", + "google.mode", "google.projectFingerprint", "google.locationFingerprint", + "openrouter.order", "openrouter.only", "openrouter.allowFallbacks", + "sidecars.vision", "sidecars.webSearch", + "mcp.maxTools", "mcp.maxSchemaBytes", "mcp.maxResultBytes", "mcp.nativeLocalExec", + "runtime.bunVersion", "runtime.platform", "runtime.arch", "runtime.streamMode", "runtime.fastMode", "runtime.effortCap", + "headers.nonCredentialBehaviorDigest", +] as const); + +const SOURCES = new Set([ + "request", "model_override", "provider_config", "registry_runtime_default", "generated_model_metadata", + "global_config", "adapter_default", "lab_forced", +]); + +const SET_ARRAY_KEYS = new Set(["modalities.input", "reasoning.efforts", "tools.choiceRestrictions", "openrouter.only"]); +const ORDERED_ARRAY_KEYS = new Set(["openrouter.order"]); +const REQUIRED_KEYS = [ + "wire.adapter", "wire.upstreamProtocol", "auth.mode", "auth.transport", "mcp.nativeLocalExec", + "runtime.bunVersion", "runtime.platform", "runtime.arch", "runtime.streamMode", "runtime.fastMode", "runtime.effortCap", ] as const; -/** Build a closed-key behavior fingerprint from lab-forced route context fields. */ -export function buildBehaviorFingerprintV1(routeContext: LabRouteContext): string { - const values: Record = { - "wire.adapter": { source: "lab_forced", value: routeContext.effectiveAdapter }, - "wire.inboundProtocol": { source: "lab_forced", value: routeContext.inboundProtocol }, - "wire.upstreamProtocol": { source: "lab_forced", value: routeContext.upstreamProtocol }, - "wire.surface": { source: "lab_forced", value: routeContext.surface }, - "route.allowPrivateNetwork": { source: "lab_forced", value: routeContext.allowPrivateNetwork === true }, - "route.labRunApproval": { source: "lab_forced", value: routeContext.labRunApproval === true }, - }; - for (const key of CLOSED_BEHAVIOR_KEYS) { - if (!(key in values)) throw new Error(`harness_failure: missing behavior key ${key}`); +function canonicalArray(key: string, value: unknown[]): unknown[] { + if (SET_ARRAY_KEYS.has(key)) { + const byBytes = value.map((item) => ({ item, bytes: jcsStringify(item) })); + const seen = new Set(); + for (const row of byBytes) { + if (seen.has(row.bytes)) throw new Error(`harness_failure: duplicate behavior array value for ${key}`); + seen.add(row.bytes); + } + byBytes.sort((a, b) => a.bytes < b.bytes ? -1 : a.bytes > b.bytes ? 1 : 0); + return byBytes.map((row) => row.item); + } + if (ORDERED_ARRAY_KEYS.has(key)) return [...value]; + throw new Error(`harness_failure: unclassified_behavior_input array for ${key}`); +} + +export function normalizeBehaviorValues(values: LabBehaviorValues): LabBehaviorValues { + if (!values || typeof values !== "object" || Array.isArray(values)) { + throw new Error("harness_failure: behavior resolver output is required"); } - const payload = { - schemaVersion: 1, - resolverVersion: 1, - values, - }; + for (const key of REQUIRED_KEYS) { + if (!(key in values)) throw new Error(`harness_failure: unclassified_behavior_input missing ${key}`); + } + const normalized: LabBehaviorValues = {}; + for (const key of Object.keys(values).sort()) { + if (!CLOSED_KEYS.has(key as never)) throw new Error(`harness_failure: unclassified_behavior_input ${key}`); + const row = values[key]; + if (!row || typeof row !== "object" || Array.isArray(row) || !SOURCES.has(row.source)) { + throw new Error(`harness_failure: invalid behavior source for ${key}`); + } + normalized[key] = { + source: row.source, + value: Array.isArray(row.value) ? canonicalArray(key, row.value) : row.value, + }; + } + return normalized; +} + +/** Hash the authoritative production resolver output; Lab performs validation/canonicalization only. */ +export function buildBehaviorFingerprintV1(values: LabBehaviorValues): string { + const payload = { schemaVersion: 1, resolverVersion: 1, values: normalizeBehaviorValues(values) }; return createHash("sha256").update(jcsStringify(payload)).digest("hex"); } diff --git a/src/lab/subject/route-subject.ts b/src/lab/subject/route-subject.ts index 9b6c22808..a5ee95ead 100644 --- a/src/lab/subject/route-subject.ts +++ b/src/lab/subject/route-subject.ts @@ -1,46 +1,53 @@ import { localFingerprint } from "../digest"; -import type { RouteSubjectV1 } from "../events/types"; +import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types"; import type { LabDestinationV1, LabRouteContext } from "../live/types"; import { buildBehaviorFingerprintV1 } from "./behavior-fingerprint"; import { readInstallationSalt } from "./installation-salt"; -const COMPAT_VERSION = "live-v1"; +function compareDependency(a: RouteDependencyV1, b: RouteDependencyV1): number { + const fields: Array = [ + "role", "providerId", "providerInstanceFingerprint", "upstreamModelId", "endpointFingerprint", + "clientModelId", "effectiveAdapter", "upstreamProtocol", "behaviorFingerprint", + ]; + for (const field of fields) { + const av = a[field]; + const bv = b[field]; + if (av < bv) return -1; + if (av > bv) return 1; + } + return 0; +} -function destinationSnapshotForFingerprint(destination: LabDestinationV1): Record { - return { - scheme: destination.scheme, - host: destination.host, - port: destination.port, - basePath: destination.basePath, - }; +function canonicalDependencies(dependencies: RouteDependencyV1[] | undefined): RouteDependencyV1[] { + const rows = (dependencies ?? []).map((row) => Object.freeze({ ...row })); + rows.sort(compareDependency); + for (let i = 1; i < rows.length; i++) { + if (compareDependency(rows[i - 1]!, rows[i]!) === 0) { + throw new Error("harness_failure: duplicate route dependency"); + } + } + return rows; } -/** Build a frozen RouteSubjectV1 from route context and immutable destination snapshot. */ +/** Build RouteSubjectV1 only from an approved immutable destination and authoritative effective resolver inputs. */ export function buildRouteSubjectV1( routeContext: LabRouteContext, destination: LabDestinationV1, configDir?: string, ): RouteSubjectV1 { + if (!routeContext.providerInstanceKey) throw new Error("harness_failure: provider instance identity is required"); + if (!/^[0-9a-f]{64}$/.test(routeContext.opencodexCompatibilityVersion)) { + throw new Error("harness_failure: invalid opencodexCompatibilityVersion"); + } + const adapterValue = routeContext.behaviorValues["wire.adapter"]?.value; + const protocolValue = routeContext.behaviorValues["wire.upstreamProtocol"]?.value; + if (adapterValue !== routeContext.effectiveAdapter || protocolValue !== routeContext.upstreamProtocol) { + throw new Error("harness_failure: behavior resolver output does not match exact route"); + } const salt = readInstallationSalt(configDir); - const endpointFingerprint = localFingerprint( - "endpoint", - destinationSnapshotForFingerprint(destination), - salt, - ); - const providerInstanceFingerprint = localFingerprint( - "providerInstance", - { - providerId: routeContext.providerId, - clientModelId: routeContext.clientModelId, - upstreamModelId: routeContext.upstreamModelId, - effectiveAdapter: routeContext.effectiveAdapter, - inboundProtocol: routeContext.inboundProtocol, - upstreamProtocol: routeContext.upstreamProtocol, - surface: routeContext.surface, - }, - salt, - ); - return Object.freeze({ + const providerInstanceFingerprint = localFingerprint("providerInstance", routeContext.providerInstanceKey, salt); + const dependencies = canonicalDependencies(routeContext.dependencies); + const subject: RouteSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "route", providerId: routeContext.providerId, @@ -51,17 +58,16 @@ export function buildRouteSubjectV1( inboundProtocol: routeContext.inboundProtocol, upstreamProtocol: routeContext.upstreamProtocol, surface: routeContext.surface, - opencodexCompatibilityVersion: COMPAT_VERSION, - behaviorFingerprint: buildBehaviorFingerprintV1(routeContext), - endpointFingerprint, - dependencies: routeContext.dependencies ?? [], - }); + opencodexCompatibilityVersion: routeContext.opencodexCompatibilityVersion, + behaviorFingerprint: buildBehaviorFingerprintV1(routeContext.behaviorValues), + endpointFingerprint: destination.fingerprint, + dependencies, + }; + return freezeRouteSubject(subject); } -/** Deep-freeze a route subject before live execution. */ +/** Deep-freeze route dependencies as well as the subject shell. */ export function freezeRouteSubject(subject: RouteSubjectV1): RouteSubjectV1 { - return Object.freeze({ - ...subject, - dependencies: [...subject.dependencies], - }); + const dependencies = subject.dependencies.map((row) => Object.freeze({ ...row })); + return Object.freeze({ ...subject, dependencies: Object.freeze(dependencies) as unknown as RouteDependencyV1[] }); } diff --git a/src/lib/lab-live-pinned-sender.ts b/src/lib/lab-live-pinned-sender.ts new file mode 100644 index 000000000..56c1e2e5b --- /dev/null +++ b/src/lib/lab-live-pinned-sender.ts @@ -0,0 +1,29 @@ +import { pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; +import type { LabCredentialLeaseV1, LabPinnedSender } from "../lab/live/types"; + +/** + * Trusted credential/transport owner. Secret headers exist only in this non-Lab module and are + * consumed directly by the pinned HTTP primitive; they are never returned to Lab code. + */ +export function createLabAuthorizedPinnedSender( + authorize: (lease: LabCredentialLeaseV1) => Promise | HeadersInit, +): LabPinnedSender { + return async (lease, destination, pinned, request, signal, limits) => { + const headers = await authorize(lease); + const url = `${destination.scheme}://${destination.host}:${destination.port}${destination.basePath}${request.path}`; + const options = { + headers, + maxBytes: limits.maxOutputBytes, + idleTimeoutMs: Math.min(limits.firstByteTimeoutMs, limits.inactivityTimeoutMs), + rejectUnauthorized: true, + context: "Lab provider response", + }; + const response = request.method === "POST" + ? await pinnedHttpPost(url, pinned, request.body ?? "", signal, options) + : await pinnedHttpGet(url, pinned, signal, options); + const body = await response.text(); + const responseHeaders: Record = {}; + response.headers.forEach((value, key) => { responseHeaders[key.toLowerCase()] = value; }); + return { status: response.status, headers: responseHeaders, body }; + }; +} From 422c1686b5793eeaf507ef006e48b238fb5fccf3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:01:44 +0200 Subject: [PATCH 05/47] fix(lab): reconcile CL-03 strict contracts --- src/lab/live/credential-lease.ts | 17 +- src/lab/live/sandbox.ts | 1 + src/lab/live/transport.ts | 1 + tests/lab-live-probe.test.ts | 311 ++++++++----------------------- tests/lab-live-sandbox.test.ts | 220 +++------------------- 5 files changed, 115 insertions(+), 435 deletions(-) diff --git a/src/lab/live/credential-lease.ts b/src/lab/live/credential-lease.ts index 7869ff490..1e557777d 100644 --- a/src/lab/live/credential-lease.ts +++ b/src/lab/live/credential-lease.ts @@ -7,6 +7,7 @@ export class LabCredentialError extends Error { interface LeaseState { destinationFingerprint: string; transportId: string; remaining: number } const LEASE_STATE = new WeakMap(); +type InternalLease = LabCredentialLeaseV1 & { toJSON(): never }; export interface CreateLeaseOptions { destination: LabDestinationV1; @@ -14,22 +15,22 @@ export interface CreateLeaseOptions { budget?: number; } -/** Trusted credential owner may attach its own secret state to this opaque capability. */ +/** Trusted credential owner may associate secret state with this opaque capability outside Lab. */ export function createCredentialLease(opts: CreateLeaseOptions): LabCredentialLeaseV1 { const budget = opts.budget ?? 1; if (!Number.isInteger(budget) || budget <= 0) throw new LabCredentialError("invalid credential lease budget", "harness_failure"); - const lease = { - [LAB_CREDENTIAL_LEASE]: true as const, - get remainingRequests() { return LEASE_STATE.get(lease)?.remaining ?? 0; }, - consume() { + let lease!: InternalLease; + lease = { + [LAB_CREDENTIAL_LEASE]: true, + get remainingRequests(): number { return LEASE_STATE.get(lease)?.remaining ?? 0; }, + consume(): void { const state = LEASE_STATE.get(lease); if (!state) throw new LabCredentialError("unknown credential lease", "harness_failure"); if (state.remaining <= 0) throw new LabCredentialError("credential lease exhausted", "budget_exhausted"); state.remaining -= 1; }, - toString() { return "[LabCredentialLeaseV1]"; }, - toJSON() { throw new LabCredentialError("credential lease is non-serializable", "harness_failure"); }, - } satisfies LabCredentialLeaseV1 & { toString(): string; toJSON(): never }; + toJSON(): never { throw new LabCredentialError("credential lease is non-serializable", "harness_failure"); }, + }; LEASE_STATE.set(lease, { destinationFingerprint: opts.destination.fingerprint, transportId: opts.transportId ?? "default", diff --git a/src/lab/live/sandbox.ts b/src/lab/live/sandbox.ts index b3b87abc6..ba54743bb 100644 --- a/src/lab/live/sandbox.ts +++ b/src/lab/live/sandbox.ts @@ -35,6 +35,7 @@ export function enforceSandboxLimits(state: SandboxResourceState, limits: LiveRu peakMemoryBytes: Math.max(state.peakMemoryBytes, delta.peakMemoryBytes ?? process.memoryUsage().rss), }; if (next.childProcesses > limits.maxChildProcesses) throw new LabSandboxError("child process limit exceeded", "child_process_limit"); + if (next.peakMemoryBytes > limits.maxMemoryBytes) throw new LabSandboxError("runner resident memory limit exceeded", "memory_limit"); if (next.requests > limits.maxRequests) throw new LabSandboxError("request limit exceeded", "request_limit"); if (next.inputBytes > limits.maxInputBytes) throw new LabSandboxError("input byte limit exceeded", "input_byte_limit"); if (next.outputBytes > limits.maxOutputBytes) throw new LabSandboxError("output byte limit exceeded", "output_byte_limit"); diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index d32811e28..55a711a26 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -27,6 +27,7 @@ export function createMockTransport(opts: MockTransportOptions = {}): LabTranspo const ok = typeof entry.matchPath === "string" ? req.path.includes(entry.matchPath) : entry.matchPath.test(req.path); if (!ok) throw new TransportError("harness_failure", "mock path mismatch"); } + if (entry.status >= 300 && entry.status < 400) throw new TransportError("redirect_blocked", "redirect response"); return { status: entry.status, headers: { "content-type": "application/json", ...(entry.headers ?? {}) }, body: entry.body }; }, }; diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts index 91950f729..a111d5f6e 100644 --- a/tests/lab-live-probe.test.ts +++ b/tests/lab-live-probe.test.ts @@ -3,283 +3,122 @@ import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { - CL03_LIVE_SUITES, - createMockTransport, - discoverLiveScenarios, - evaluateAllApplicableRequiredPassV1, - executeInertTool, - expandLiveScenario, - invokeMcpStub, - listInertToolDefinitions, - listLiveScenarioIds, - loadLiveCaseAuthority, - observationFromLiveResult, - persistLiveResult, - rebuildLabProjection, - registerMcpStubTool, - routeSubjectApplicableToRequirements, - runLiveScenario, - runLiveSuite, - scenarioManifestDigest, - subjectIdForSubject, + CL03_LIVE_SUITES, createMockTransport, discoverLiveScenarios, evaluateAllApplicableRequiredPassV1, + executeInertTool, expandLiveScenario, invokeMcpStub, listInertToolDefinitions, listLiveScenarioIds, + loadLiveCaseAuthority, observationFromLiveResult, persistLiveResult, rebuildLabProjection, + registerMcpStubTool, routeSubjectApplicableToRequirements, runLiveScenario, runLiveSuite, + scenarioManifestDigest, subjectIdForSubject, } from "../src/lab"; import { replayLabLedger } from "../src/lab/ledger/store"; import { clearMcpStub } from "../src/lab/live/mcp-loopback"; import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; -import type { LabRouteContext } from "../src/lab/live/types"; +import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; import type { RouteSubjectV1 } from "../src/lab/events/types"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; const HOMES: string[] = []; +function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-probe-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } +afterEach(() => { for (const dir of HOMES.splice(0)) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } delete process.env.OPENCODEX_HOME; clearMcpStub(); }); -function tempHome(): string { - const dir = join(tmpdir(), `ocx-lab-probe-${process.pid}-${Math.random().toString(16).slice(2)}`); - mkdirSync(dir, { recursive: true, mode: 0o700 }); - HOMES.push(dir); - return dir; +function behavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: adapter }, "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, + "auth.mode": { source: "provider_config", value: "api_key" }, "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; } -afterEach(() => { - for (const dir of HOMES.splice(0)) { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } - } - delete process.env.OPENCODEX_HOME; - clearMcpStub(); -}); - function mockRoute(overrides: Partial = {}): LabRouteContext { - return { - providerId: "fixture-provider", - clientModelId: "fixture-model", - upstreamModelId: "fixture-model", - effectiveAdapter: "openai-chat", - inboundProtocol: "openai-responses", - upstreamProtocol: "openai-chat", - surface: "responses-http", - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - allowPrivateNetwork: false, + const base = { + providerId: "fixture-provider", providerInstanceKey: "fixture-provider-instance", clientModelId: "fixture-model", upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", + baseUrl: "https://api.example.com/v1", opencodexCompatibilityVersion: "a".repeat(64), labRunApproval: true, allowPrivateNetwork: false, requiredClaims: ["tools", "image", "reasoning"], - ...overrides, + availableHarnessFeatures: ["live_transport", "inert_tools", "adapter_vector", "reasoning_replay", "synthetic_image", "in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub"], }; + const merged = { ...base, ...overrides }; + return { ...merged, behaviorValues: overrides.behaviorValues ?? behavior(merged.effectiveAdapter, merged.upstreamProtocol) }; } function transportForCase(caseRecord: ReturnType[number]) { - const entries = []; - if (caseRecord.initiatingRequest) { - entries.push({ - status: 200, - headers: { "content-type": caseRecord.fixture.mediaType }, - body: caseRecord.fixture.bytesUtf8, - }); - } - return createMockTransport({ entries }); + return createMockTransport({ entries: caseRecord.initiatingRequest ? [{ status: 200, headers: { "content-type": caseRecord.fixture.mediaType }, body: caseRecord.fixture.bytesUtf8 }] : [] }); +} + +function reasoningObservation(): NormalizedObservation { + return { + client: { request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "" } }, + upstream: { requests: [ + { status: 200, headers: {}, json: {}, rawBytes: 0 }, + { status: 200, headers: {}, json: { input: [{ signature: "sig_fixture" }] }, rawBytes: 0 }, + ], responses: [] }, + process: { exitCode: null }, verifiers: {}, + }; } describe("CL-03 live probe harness", () => { test("loads frozen 10-case live authority with reproducible digests", () => { - const authority = loadLiveCaseAuthority(); - expect(authority.cases.length).toBe(10); - const ids = authority.cases.map((c) => c.id); - expect(ids).toContain("responses-core.live.basic-turn"); - expect(ids).toContain("mcp-core.live.synthetic-tool"); - const expandedA = expandLiveScenario(authority.cases[0]!, authority); - const expandedB = expandLiveScenario(authority.cases[0]!, authority); - expect(scenarioManifestDigest(expandedA)).toBe(scenarioManifestDigest(expandedB)); - }); - - test("discovers all CL-03 live suites", () => { - const authority = loadLiveCaseAuthority(); - const scenarios = discoverLiveScenarios(authority, CL03_LIVE_SUITES); - expect(scenarios.length).toBe(10); - expect(listLiveScenarioIds()).toHaveLength(10); + const authority = loadLiveCaseAuthority(); expect(authority.cases.length).toBe(10); + expect(authority.cases.map((c) => c.id)).toContain("responses-core.live.basic-turn"); + expect(authority.cases.map((c) => c.id)).toContain("mcp-core.live.synthetic-tool"); + expect(scenarioManifestDigest(expandLiveScenario(authority.cases[0]!, authority))).toBe(scenarioManifestDigest(expandLiveScenario(authority.cases[0]!, authority))); }); - test("runs live suite with mock transport and no external network", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const route = mockRoute(); - const summary = await runLiveSuite(route, ["responses-core", "chat-core"], { - configDir: home, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - transport: createMockTransport({ - entries: [ - { status: 200, body: authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!.fixture.bytesUtf8 }, - { status: 200, body: authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!.fixture.bytesUtf8 }, - ], - }), - }); - expect(summary.total).toBe(2); - expect(summary.results.every((r) => r.routeSubject.subjectKind === "route")).toBe(true); - }); + test("discovers all CL-03 live suites", () => { const authority = loadLiveCaseAuthority(); expect(discoverLiveScenarios(authority, CL03_LIVE_SUITES)).toHaveLength(10); expect(listLiveScenarioIds()).toHaveLength(10); }); - test("inert tool round-trips validate args and return static results", () => { - expect(listInertToolDefinitions().map((t) => t.name)).toContain("lookup"); - expect(executeInertTool("lookup", { q: "x" }).output).toBe("RESULT"); - expect(() => executeInertTool("lookup", { q: 1 })).toThrow(); + test("runs only applicable live suite scenarios with an injected test transport", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); + const summary = await runLiveSuite(mockRoute(), ["responses-core", "chat-core"], { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!.fixture.bytesUtf8 }] }) }); + expect(summary.total).toBe(1); expect(summary.results[0]?.routeSubject?.subjectKind).toBe("route"); }); - test("MCP lab stub only — no user mcpServers", () => { - registerMcpStubTool({ namespace: "lab", name: "echo" }); - const result = invokeMcpStub("lab", "echo", { x: 1 }, { content: [{ type: "text", text: "ECHO" }] }); - expect(result.content[0]?.text).toBe("ECHO"); - }); + test("inert tool round-trips validate args and return static results", () => { expect(listInertToolDefinitions().map((t) => t.name)).toContain("lookup"); expect(executeInertTool("lookup", { q: "x" }).output).toBe("RESULT"); expect(() => executeInertTool("lookup", { q: 1 })).toThrow(); }); + test("MCP lab stub only", () => { registerMcpStubTool({ namespace: "lab", name: "echo" }); expect(invokeMcpStub("lab", "echo", { x: 1 }, { content: [{ type: "text", text: "ECHO" }] }).content[0]?.text).toBe("ECHO"); }); test("raw URLs and secrets never appear in persisted live evidence", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; - const result = await runLiveScenario(caseRecord, mockRoute(), { - configDir: home, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - transport: transportForCase(caseRecord), - }); - const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); - const serialized = JSON.stringify(event); - expect(serialized).not.toContain("api.example.com"); - expect(serialized).not.toContain("https://"); - expect(serialized).not.toContain("apiKey"); - expect(serialized).not.toContain("fixture-key"); - expect(event.evidenceLayer).toBe("live_route_compatibility"); - expect(event.executionMode).toBe("live"); - expect(event.subject.subjectKind).toBe("route"); + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); const serialized = JSON.stringify(event); + expect(serialized).not.toContain("api.example.com"); expect(serialized).not.toContain("https://"); expect(serialized).not.toContain("apiKey"); expect(event.evidenceLayer).toBe("live_route_compatibility"); expect(event.executionMode).toBe("live"); }); test("JSONL persistence and SQLite rebuild for live evidence", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const caseRecord = authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!; - const result = await runLiveScenario(caseRecord, mockRoute({ - inboundProtocol: "openai-responses", - upstreamProtocol: "openai-chat", - surface: "responses-sse", - }), { - configDir: home, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - transport: transportForCase(caseRecord), - }); - persistLiveResult(result, caseRecord, authority, { configDir: home }); - const projection = rebuildLabProjection(home); - expect(projection.events).toBeGreaterThan(0); - expect(projection.corruptions).toHaveLength(0); - const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); - expect(replay.events.length).toBeGreaterThan(0); + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!; + const route = mockRoute({ effectiveAdapter: "openai-chat", upstreamProtocol: "openai-chat", surface: "responses-sse" }); + const result = await runLiveScenario(caseRecord, route, { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); + persistLiveResult(result, caseRecord, authority, { configDir: home }); const projection = rebuildLabProjection(home); expect(projection.events).toBeGreaterThan(0); expect(projection.corruptions).toHaveLength(0); expect(replayLabLedger(join(home, "lab", "compatibility.jsonl")).events.length).toBeGreaterThan(0); }); test("route subject applicability and cross-route evidence reuse prevention", () => { - const authority = loadLiveCaseAuthority(); - const subjectA: RouteSubjectV1 = { - subjectSchemaVersion: 1, - subjectKind: "route", - providerId: "provider-a", - providerInstanceFingerprint: "a".repeat(64), - clientModelId: "model-a", - upstreamModelId: "model-a", - effectiveAdapter: "openai-responses", - inboundProtocol: "openai-responses", - upstreamProtocol: "openai-responses", - surface: "responses-http", - opencodexCompatibilityVersion: "live-v1", - behaviorFingerprint: "b".repeat(64), - endpointFingerprint: "c".repeat(64), - dependencies: [], - }; - const subjectB: RouteSubjectV1 = { ...subjectA, providerId: "provider-b", endpointFingerprint: "d".repeat(64) }; - expect(subjectIdForSubject(subjectA)).not.toBe(subjectIdForSubject(subjectB)); - const caseRecord = authority.cases[0]!; - const requirements = caseRecord.requirements; - expect(routeSubjectApplicableToRequirements(requirements, subjectA, [])).toBe(true); - expect(routeSubjectApplicableToRequirements( - { ...requirements, inboundProtocols: ["anthropic-messages"] }, - subjectA, - [], - )).toBe(false); + const subjectA: RouteSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "route", providerId: "provider-a", providerInstanceFingerprint: "a".repeat(64), clientModelId: "model-a", upstreamModelId: "model-a", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; + const subjectB = { ...subjectA, providerId: "provider-b", endpointFingerprint: "d".repeat(64) } as RouteSubjectV1; expect(subjectIdForSubject(subjectA)).not.toBe(subjectIdForSubject(subjectB)); + const requirements = loadLiveCaseAuthority().cases[0]!.requirements; expect(routeSubjectApplicableToRequirements(requirements, subjectA, [])).toBe(true); expect(routeSubjectApplicableToRequirements({ ...requirements, inboundProtocols: ["anthropic-messages"] }, subjectA, [])).toBe(false); }); - test("freshness and contradictory evidence semantics for live suites", () => { - const authority = loadLiveCaseAuthority(); - const suite = expandLiveSuiteManifest("responses-core", authority); - expect(suite.freshness.maxAgeMs).toBe(604800000); - expect(suite.evidenceLayer).toBe("live_route_compatibility"); - const routeSubject: RouteSubjectV1 = { - subjectSchemaVersion: 1, - subjectKind: "route", - providerId: "provider-a", - providerInstanceFingerprint: "a".repeat(64), - clientModelId: "model-a", - upstreamModelId: "model-a", - effectiveAdapter: "openai-responses", - inboundProtocol: "openai-responses", - upstreamProtocol: "openai-responses", - surface: "responses-http", - opencodexCompatibilityVersion: "live-v1", - behaviorFingerprint: "b".repeat(64), - endpointFingerprint: "c".repeat(64), - dependencies: [], - }; - const evalEmpty = evaluateAllApplicableRequiredPassV1(suite, [], "live", { - subject: routeSubject, - loadScenarioRequirements: () => ({ - inboundProtocols: ["openai-responses"], - upstreamProtocols: ["openai-responses"], - surfaces: ["responses-http"], - requiredClaims: [], - freshness: { maxAgeMs: 604800000 }, - }), - }); - expect(evalEmpty.canVerify).toBe(false); - expect(evalEmpty.missingRequiredScenarioIds.length).toBeGreaterThan(0); + test("freshness semantics for live suites", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); expect(suite.freshness.maxAgeMs).toBe(604800000); + const subject: RouteSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "route", providerId: "p", providerInstanceFingerprint: "a".repeat(64), clientModelId: "m", upstreamModelId: "m", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; + const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject, loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + expect(result.canVerify).toBe(false); expect(result.missingRequiredScenarioIds.length).toBeGreaterThan(0); }); - test("timeouts classify as blockers not degraded/unsupported", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const caseRecord = authority.cases[0]!; - const result = await runLiveScenario(caseRecord, mockRoute({ - upstreamProtocol: "openai-responses", - }), { - configDir: home, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - transport: createMockTransport({ - entries: [{ status: 200, body: "{}", error: "total_timeout" }], - }), - }); + test("timeouts classify as blockers", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const scenario = loadLiveCaseAuthority().cases[0]!; + const result = await runLiveScenario(scenario, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: "{}", error: "total_timeout" }] }) }); expect(result.classification).toBe("timeout"); - expect(result.classification).not.toBe("protocol_failure"); }); - test("auth failure is a blocker", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const caseRecord = authority.cases[0]!; - const result = await runLiveScenario(caseRecord, mockRoute({ - upstreamProtocol: "openai-responses", - }), { - configDir: home, - authOutcome: "blocked", - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - }); - expect(result.classification).toBe("authentication_blocked"); + test("HTTP auth failure is an exact-route blocker", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const scenario = loadLiveCaseAuthority().cases[0]!; + const result = await runLiveScenario(scenario, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 401, body: "" }] }) }); + expect(result.classification).toBe("authentication_blocked"); expect(result.routeSubject?.endpointFingerprint).toHaveLength(64); }); - test("reasoning replay exposes no private material in persisted evidence", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const authority = loadLiveCaseAuthority(); - const caseRecord = authority.cases.find((c) => c.id === "reasoning-core.live.replay")!; - const result = await runLiveScenario(caseRecord, mockRoute({ - upstreamProtocol: "openai-responses", - requiredClaims: ["reasoning"], - }), { - configDir: home, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - transport: createMockTransport({ entries: [{ status: 200, body: "{}" }] }), - }); - const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); - expect(JSON.stringify(event)).not.toContain("PLAN"); + test("reasoning replay persists no private reasoning material", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const scenario = authority.cases.find((c) => c.id === "reasoning-core.live.replay")!; + const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: async () => reasoningObservation() }); + expect(result.passed).toBe(true); const { event } = observationFromLiveResult(result, scenario, authority, { configDir: home }); expect(JSON.stringify(event)).not.toContain("PLAN"); }); }); diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index cbc27efb8..8a35d3ada 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -1,211 +1,49 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { - assertDestinationAddressSet, - assertHostSniMatch, - assertLeaseScope, - buildRouteSubjectV1, - createCredentialLease, - createLabDestination, - createMockTransport, - createSandboxResourceState, - enforceSandboxLimits, - freezeRouteSubject, - LabCredentialError, - LabDestinationError, - LabSandboxError, - prepareLiveSandbox, - rejectProxyEnvironment, - TransportError, - classifyTransportError, + assertDestinationAddressSet, assertHostSniMatch, assertLeaseScope, buildRouteSubjectV1, classifyTransportError, + createCredentialLease, createLabDestination, createMockTransport, createSandboxResourceState, enforceSandboxLimits, + freezeRouteSubject, LabCredentialError, LabDestinationError, LabSandboxError, prepareLiveSandbox, rejectProxyEnvironment, TransportError, } from "../src/lab"; -import type { LabRouteContext } from "../src/lab/live/types"; +import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; const HOMES: string[] = []; +function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } +afterEach(() => { for (const dir of HOMES.splice(0)) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } delete process.env.OPENCODEX_HOME; }); -function tempHome(): string { - const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); - mkdirSync(dir, { recursive: true, mode: 0o700 }); - HOMES.push(dir); - return dir; +function behavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { "wire.adapter": { source: "lab_forced", value: adapter }, "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, "auth.mode": { source: "provider_config", value: "api_key" }, "auth.transport": { source: "provider_config", value: "authorization_bearer" }, "mcp.nativeLocalExec": { source: "lab_forced", value: false }, "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, "runtime.platform": { source: "lab_forced", value: process.platform }, "runtime.arch": { source: "lab_forced", value: process.arch }, "runtime.streamMode": { source: "lab_forced", value: "auto" }, "runtime.fastMode": { source: "lab_forced", value: false }, "runtime.effortCap": { source: "lab_forced", value: null }, "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) } }; } - -afterEach(() => { - for (const dir of HOMES.splice(0)) { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } - } - delete process.env.OPENCODEX_HOME; -}); - function baseRoute(overrides: Partial = {}): LabRouteContext { - return { - providerId: "fixture-provider", - clientModelId: "fixture-model", - upstreamModelId: "fixture-model", - effectiveAdapter: "openai-chat", - inboundProtocol: "openai-responses", - upstreamProtocol: "openai-chat", - surface: "responses-http", - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - allowPrivateNetwork: false, - requiredClaims: [], - ...overrides, - }; + const base = { providerId: "fixture-provider", providerInstanceKey: "fixture-instance", clientModelId: "fixture-model", upstreamModelId: "fixture-model", effectiveAdapter: "openai-chat", inboundProtocol: "openai-responses", upstreamProtocol: "openai-chat", surface: "responses-http", baseUrl: "https://api.example.com/v1", opencodexCompatibilityVersion: "a".repeat(64), labRunApproval: true, allowPrivateNetwork: false, requiredClaims: [] as string[], availableHarnessFeatures: ["live_transport"] }; + const merged = { ...base, ...overrides }; return { ...merged, behaviorValues: overrides.behaviorValues ?? behavior(merged.effectiveAdapter, merged.upstreamProtocol) }; } +const limits = { totalTimeoutMs: 120000, connectTimeoutMs: 10000, firstByteTimeoutMs: 30000, inactivityTimeoutMs: 30000, maxRequests: 16, maxInputBytes: 8388608, maxOutputBytes: 16777216, maxOutputTokens: 32768, maxToolCalls: 32, maxMemoryBytes: 536870912, maxChildProcesses: 0, maxArtifacts: 16, perArtifactBytes: 262144, aggregateArtifactBytes: 1048576 }; describe("CL-03 live sandbox security seams", () => { - test("rejects proxy environment variables", () => { - expect(() => rejectProxyEnvironment({ HTTP_PROXY: "http://evil" })).toThrow(LabSandboxError); - expect(() => rejectProxyEnvironment({ no_proxy: "localhost" })).toThrow(LabSandboxError); - }); - - test("prepareLiveSandbox allows only TZ=UTC and NO_COLOR=1", () => { - const env = prepareLiveSandbox({}); - expect(env).toEqual({ TZ: "UTC", NO_COLOR: "1" }); - }); - - test("DNS/address mismatch fails before credentials", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const destination = await createLabDestination({ - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - configDir: home, - }); - expect(() => assertDestinationAddressSet(destination, [{ address: "1.2.3.4", family: 4 }])) - .toThrow(LabDestinationError); - expect(() => createCredentialLease({ destination, authOutcome: "blocked" })) - .toThrow(LabCredentialError); - }); - - test("host/SNI mismatch fails before credentials", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const destination = await createLabDestination({ - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - configDir: home, - }); - expect(() => assertHostSniMatch(destination, "evil.example.com")) - .toThrow(LabDestinationError); - }); + test("rejects proxy environment variables", () => { expect(() => rejectProxyEnvironment({ HTTP_PROXY: "http://evil" })).toThrow(LabSandboxError); expect(() => rejectProxyEnvironment({ no_proxy: "localhost" })).toThrow(LabSandboxError); }); + test("prepareLiveSandbox allows only TZ=UTC and NO_COLOR=1", () => { expect(prepareLiveSandbox({})).toEqual({ TZ: "UTC", NO_COLOR: "1" }); }); - test("metadata and link-local destinations are blocked", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - await expect(createLabDestination({ - baseUrl: "http://169.254.169.254/latest/meta-data", - labRunApproval: true, - configDir: home, - })).rejects.toThrow(LabDestinationError); + test("DNS/address mismatch and lease scope mismatch fail closed", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); + expect(() => assertDestinationAddressSet(destination, [{ address: "1.2.3.4", family: 4 }])).toThrow(LabDestinationError); + const other = await createLabDestination({ baseUrl: "https://api.example.com/v2", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); + const lease = createCredentialLease({ destination, budget: 1 }); expect(() => assertLeaseScope(lease, other)).toThrow(LabCredentialError); }); - test("private network requires permission and lab approval", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - await expect(createLabDestination({ - baseUrl: "http://127.0.0.1:11434/v1", - configDir: home, - })).rejects.toThrow(LabDestinationError); - const ok = await createLabDestination({ - baseUrl: "http://127.0.0.1:11434/v1", - allowPrivateNetwork: true, - labRunApproval: true, - resolve: async () => [{ address: "127.0.0.1", family: 4 }], - configDir: home, - }); - expect(ok.privateNetwork).toBe(true); - }); - - test("redirects fail closed in mock transport", async () => { - const transport = createMockTransport({ - entries: [{ status: 302, body: "", headers: { location: "https://evil.example.com" } }], - }); - await expect(transport.request({ method: "GET", path: "/" })).rejects.toThrow(TransportError); - }); - - test("credential lease scope is destination-bound", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const destination = await createLabDestination({ - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - configDir: home, - }); - const lease = createCredentialLease({ destination, budget: 1 }); - assertLeaseScope(lease, destination); - lease.consume(); - expect(() => lease.consume()).toThrow(LabCredentialError); - }); + test("host/SNI mismatch fails closed", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); expect(() => assertHostSniMatch(destination, "evil.example.com")).toThrow(LabDestinationError); }); + test("metadata and link-local destinations are blocked", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://169.254.169.254/latest/meta-data", labRunApproval: true, configDir: home })).rejects.toThrow(LabDestinationError); }); + test("private network requires permission and lab approval", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", configDir: home })).rejects.toThrow(LabDestinationError); const ok = await createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", allowPrivateNetwork: true, labRunApproval: true, resolve: async () => [{ address: "127.0.0.1", family: 4 }], configDir: home }); expect(ok.privateNetwork).toBe(true); }); + test("redirects fail closed in mock transport", async () => { const transport = createMockTransport({ entries: [{ status: 302, body: "", headers: { location: "https://evil.example.com" } }] }); await expect(transport.request({ method: "GET", path: "/" })).rejects.toThrow(TransportError); }); - test("auth/quota/network/transient classify as blockers not compatibility failures", () => { - expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification) - .toBe("authentication_blocked"); - expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification) - .toBe("quota_blocked"); - expect(classifyTransportError(new TransportError("network_blocked", "net")).classification) - .toBe("network_failure"); - expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification) - .toBe("provider_transient"); - expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification) - .toBe("timeout"); - }); + test("credential lease is destination-bound, bounded, and non-serializable", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const lease = createCredentialLease({ destination, budget: 1 }); assertLeaseScope(lease, destination); expect(() => JSON.stringify(lease)).toThrow(LabCredentialError); lease.consume(); expect(() => lease.consume()).toThrow(LabCredentialError); }); + test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); - test("resource ceilings enforce child process limit", () => { - const state = createSandboxResourceState(); - const limits = { - totalTimeoutMs: 120000, - connectTimeoutMs: 10000, - firstByteTimeoutMs: 30000, - inactivityTimeoutMs: 30000, - maxRequests: 16, - maxInputBytes: 8388608, - maxOutputBytes: 16777216, - maxOutputTokens: 32768, - maxToolCalls: 32, - maxMemoryBytes: 536870912, - maxChildProcesses: 0, - maxArtifacts: 16, - perArtifactBytes: 262144, - aggregateArtifactBytes: 1048576, - }; - expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })) - .toThrow(LabSandboxError); - }); + test("resource ceilings enforce child-process and memory limits", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); const state2 = createSandboxResourceState(); expect(() => enforceSandboxLimits(state2, { ...limits, maxMemoryBytes: 1 })).toThrow(LabSandboxError); }); - test("endpoint fingerprint comes from immutable destination snapshot only", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const destination = await createLabDestination({ - baseUrl: "https://api.example.com/v1/custom", - labRunApproval: true, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - configDir: home, - }); - const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); - expect(subject.endpointFingerprint).toHaveLength(64); - expect(JSON.stringify(subject)).not.toContain("api.example.com"); - expect(JSON.stringify(subject)).not.toContain("https://"); - }); - - test("route subject stability across credential rotation", async () => { - const home = tempHome(); - process.env.OPENCODEX_HOME = home; - const destination = await createLabDestination({ - baseUrl: "https://api.example.com/v1", - labRunApproval: true, - resolve: async () => [{ address: "93.184.216.34", family: 4 }], - configDir: home, - }); - const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1" }), destination, home)); - createCredentialLease({ destination, budget: 1 }).consume(); - const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1" }), destination, home)); - expect(subjectA).toEqual(subjectB); - }); + test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); + test("route subject stability across credential rotation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); createCredentialLease({ destination, budget: 1 }).consume(); const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); expect(subjectA).toEqual(subjectB); }); }); From 41c5c1dd21f5af9d8fac3392ca171147db79a3be Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:19:30 +0200 Subject: [PATCH 06/47] fix(lab): gate live evidence on trusted execution --- src/lab/live/executor.ts | 25 ++++++++------ src/lab/live/sandbox.ts | 19 ++++++++--- src/lab/live/transport.ts | 2 +- src/lab/live/types.ts | 31 +++++++++++++++++- src/lab/observe/from-live.ts | 1 + src/lib/lab-live-execution-authority.ts | 41 +++++++++++++++++++++++ tests/lab-live-probe.test.ts | 43 +++++++++++++++++-------- tests/lab-live-sandbox.test.ts | 12 +++++-- 8 files changed, 142 insertions(+), 32 deletions(-) create mode 100644 src/lib/lab-live-execution-authority.ts diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index 24ff020ed..4b31f07c4 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -3,17 +3,18 @@ import { emptyObservation, finalizeObservation, setClientResponse } from "../con import { normalizeSseBytes } from "../conformance/sse-normalize"; import type { CaseRecord, FailureClassification, FailureRule, NormalizedEvent, NormalizedObservation } from "../conformance/types"; import type { RouteSubjectV1 } from "../events/types"; +import { isTrustedLabRouteExecutor } from "../../lib/lab-live-execution-authority"; import { buildRouteSubjectV1 } from "../subject/route-subject"; import { createLabDestination, LabDestinationError } from "./destination"; import { expandLiveScenario } from "./manifest"; import { createSandboxResourceState, enforceSandboxLimits, LabSandboxError, prepareLiveSandbox } from "./sandbox"; import { classifyTransportError, TransportError } from "./transport"; -import type { DnsResolver, LabRouteContext, LabRouteExecutor, LabTransport, LiveRunConfig, LiveScenarioRunResult } from "./types"; +import type { DnsResolver, LabRouteContext, LabTransport, LiveExecutionAuthority, LiveRunConfig, LiveScenarioRunResult, TrustedLabRouteExecutor } from "./types"; export interface LiveExecutorOptions { - /** Trusted exact-route executor for production use; owns adapter + credential plumbing. */ - routeExecutor?: LabRouteExecutor; - /** Injectable byte transport used by deterministic tests. Never created implicitly. */ + /** Branded exact-route capability for evidence-eligible production execution. */ + routeExecutor?: TrustedLabRouteExecutor; + /** Test-only byte transport. Results are deliberately not evidence-eligible. */ transport?: LabTransport; resolve?: DnsResolver; configDir?: string; @@ -125,7 +126,7 @@ function responsesObservation(body: string, status: number): NormalizedObservati return observation; } -function normalizeTransportObservation(caseRecord: CaseRecord, route: LabRouteContext, response: { status: number; body: string }): NormalizedObservation { +function normalizeTransportObservation(route: LabRouteContext, response: { status: number; body: string }): NormalizedObservation { const observation = route.upstreamProtocol === "openai-chat" ? chatObservation(response.body, response.status) : responsesObservation(response.body, response.status); if (route.surface.startsWith("anthropic-") && observation.client.response.terminal === "completed") { setClientResponse(observation, { terminal: "message_stop" }); @@ -148,7 +149,8 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR const diagnostics: string[] = []; const startedAt = Date.now(); let routeSubject: RouteSubjectV1 | undefined; - const complete = (partial: Omit): LiveScenarioRunResult => ({ ...partial, startedAt, completedAt: Math.max(startedAt, Date.now()) }); + let executionAuthority: LiveExecutionAuthority = "none"; + const complete = (partial: Omit): LiveScenarioRunResult => ({ ...partial, executionAuthority, startedAt, completedAt: Math.max(startedAt, Date.now()) }); try { const environment = prepareLiveSandbox(opts.env); if (!isLiveCaseApplicableToRoute(caseRecord, routeContext)) { @@ -166,8 +168,11 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR const state = createSandboxResourceState(); let observation: NormalizedObservation; if (opts.routeExecutor) { - observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!({ routeContext, destination, routeSubject: routeSubject!, scenarioId: caseRecord.id, initiatingRequest: caseRecord.initiatingRequest?.bytesUtf8, limits, signal, environment })); + if (!isTrustedLabRouteExecutor(opts.routeExecutor)) throw new TransportError("untrusted_route_executor", "untrusted route executor capability"); + executionAuthority = "trusted_route"; + observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: caseRecord.id, initiatingRequest: caseRecord.initiatingRequest?.bytesUtf8, limits, signal, environment })); } else if (opts.transport && caseRecord.initiatingRequest) { + executionAuthority = "test_transport"; const body = caseRecord.initiatingRequest.bytesUtf8; const inputBytes = new TextEncoder().encode(body).byteLength; enforceSandboxLimits(state, limits, { requests: 1, inputBytes }); @@ -177,10 +182,10 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`); const outputBytes = new TextEncoder().encode(response.body).byteLength; - enforceSandboxLimits(state, limits, { outputBytes, outputTokens: Math.ceil(outputBytes / 4), peakMemoryBytes: process.memoryUsage().rss }); - observation = normalizeTransportObservation(caseRecord, routeContext, response); + enforceSandboxLimits(state, limits, { outputBytes, outputTokens: Math.ceil(outputBytes / 4) }); + observation = normalizeTransportObservation(routeContext, response); } else { - throw new TransportError("harness_failure", caseRecord.initiatingRequest ? "live_transport_required" : "live_route_executor_required"); + throw new TransportError("live_transport_required", caseRecord.initiatingRequest ? "live transport or trusted route executor required" : "trusted route executor required"); } const assertionResults = evaluateAssertions(caseRecord.assertions, observation); const requiredFailures = assertionResults.filter((row) => row.required && !row.passed); diff --git a/src/lab/live/sandbox.ts b/src/lab/live/sandbox.ts index ba54743bb..7399a31c9 100644 --- a/src/lab/live/sandbox.ts +++ b/src/lab/live/sandbox.ts @@ -18,24 +18,33 @@ export function rejectProxyEnvironment(env: NodeJS.ProcessEnv = process.env): vo export function labSandboxEnvironment(): Readonly> { return Object.freeze({ ...ALLOWED_ENV }); } export interface SandboxResourceState { - requests: number; inputBytes: number; outputBytes: number; outputTokens: number; toolCalls: number; - artifacts: number; artifactBytes: number; childProcesses: number; peakMemoryBytes: number; + requests: number; + inputBytes: number; + outputBytes: number; + outputTokens: number; + toolCalls: number; + artifacts: number; + artifactBytes: number; + childProcesses: number; } export function createSandboxResourceState(): SandboxResourceState { - return { requests: 0, inputBytes: 0, outputBytes: 0, outputTokens: 0, toolCalls: 0, artifacts: 0, artifactBytes: 0, childProcesses: 0, peakMemoryBytes: process.memoryUsage().rss }; + return { requests: 0, inputBytes: 0, outputBytes: 0, outputTokens: 0, toolCalls: 0, artifacts: 0, artifactBytes: 0, childProcesses: 0 }; } +/** + * Counter-based ceilings enforce the resources observable in this process. The hard resident-memory + * boundary is NOT inferred from the hosting Bun test/server process RSS; a trusted exact-route + * executor must attest/enforce that isolated boundary before its result is evidence-eligible. + */ export function enforceSandboxLimits(state: SandboxResourceState, limits: LiveRunConfig, delta: Partial = {}): void { const next: SandboxResourceState = { requests: state.requests + (delta.requests ?? 0), inputBytes: state.inputBytes + (delta.inputBytes ?? 0), outputBytes: state.outputBytes + (delta.outputBytes ?? 0), outputTokens: state.outputTokens + (delta.outputTokens ?? 0), toolCalls: state.toolCalls + (delta.toolCalls ?? 0), artifacts: state.artifacts + (delta.artifacts ?? 0), artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), childProcesses: state.childProcesses + (delta.childProcesses ?? 0), - peakMemoryBytes: Math.max(state.peakMemoryBytes, delta.peakMemoryBytes ?? process.memoryUsage().rss), }; if (next.childProcesses > limits.maxChildProcesses) throw new LabSandboxError("child process limit exceeded", "child_process_limit"); - if (next.peakMemoryBytes > limits.maxMemoryBytes) throw new LabSandboxError("runner resident memory limit exceeded", "memory_limit"); if (next.requests > limits.maxRequests) throw new LabSandboxError("request limit exceeded", "request_limit"); if (next.inputBytes > limits.maxInputBytes) throw new LabSandboxError("input byte limit exceeded", "input_byte_limit"); if (next.outputBytes > limits.maxOutputBytes) throw new LabSandboxError("output byte limit exceeded", "output_byte_limit"); diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index 55a711a26..bc1ced001 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -108,7 +108,7 @@ export function classifyTransportError(error: unknown): { classification: string case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit": case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit": return { classification: "budget_exhausted", secondaryCode: code }; - case "redirect_blocked": case "destination_mismatch": case "host_sni_mismatch": case "harness_failure": + case "live_transport_required": case "untrusted_route_executor": case "redirect_blocked": case "destination_mismatch": case "host_sni_mismatch": case "harness_failure": return { classification: "harness_failure", secondaryCode: code }; default: return { classification: "harness_failure", secondaryCode: "execution_error" }; } diff --git a/src/lab/live/types.ts b/src/lab/live/types.ts index 693f47db7..d00b7c6fb 100644 --- a/src/lab/live/types.ts +++ b/src/lab/live/types.ts @@ -92,6 +92,8 @@ export type TransportErrorCode = | "artifact_byte_limit" | "memory_limit" | "child_process_limit" + | "live_transport_required" + | "untrusted_route_executor" | "harness_failure"; export interface LabTransportRequest { @@ -150,9 +152,35 @@ export interface LabRouteExecutorInput { environment: Readonly>; } -/** Trusted exact-route execution seam. It must use existing route/adapters and return client-observed normalization. */ +/** Trusted exact-route execution function. The wrapper capability is branded outside Lab. */ export type LabRouteExecutor = (input: LabRouteExecutorInput) => Promise; +export const REQUIRED_LAB_SANDBOX_BOUNDARIES = [ + "destination_pinning", + "credential_binding", + "wall_clock", + "connect_timeout", + "first_byte_timeout", + "inactivity_timeout", + "request_budget", + "input_byte_budget", + "output_byte_budget", + "output_token_budget", + "tool_call_budget", + "memory_limit", + "child_process_limit", + "artifact_budget", +] as const; +export type LabSandboxBoundary = typeof REQUIRED_LAB_SANDBOX_BOUNDARIES[number]; + +/** Opaque-ish internal capability; runtime trust is established by a WeakSet in src/lib. */ +export interface TrustedLabRouteExecutor { + execute(input: LabRouteExecutorInput): Promise; + readonly enforcedBoundaries: readonly LabSandboxBoundary[]; +} + +export type LiveExecutionAuthority = "trusted_route" | "test_transport" | "none"; + export interface LiveScenarioRunResult { scenarioId: string; suite: string; @@ -165,4 +193,5 @@ export interface LiveScenarioRunResult { diagnostics: string[]; routeSubject?: RouteSubjectV1; transportError?: TransportErrorCode; + executionAuthority: LiveExecutionAuthority; } diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index 0d5d190b2..ace436bd4 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -29,6 +29,7 @@ function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveO } export function observationFromLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): { event: ObservationEvent; artifacts: ReturnType[] } { + if (result.executionAuthority !== "trusted_route") throw new Error("only trusted-route execution may be persisted as live evidence"); if (!result.routeSubject) throw new Error("live evidence without an exact RouteSubjectV1 is not persistable"); const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { diff --git a/src/lib/lab-live-execution-authority.ts b/src/lib/lab-live-execution-authority.ts new file mode 100644 index 000000000..75b172d52 --- /dev/null +++ b/src/lib/lab-live-execution-authority.ts @@ -0,0 +1,41 @@ +import type { + LabRouteExecutor, + LabSandboxBoundary, + TrustedLabRouteExecutor, +} from "../lab/live/types"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES } from "../lab/live/types"; + +const TRUSTED_EXECUTORS = new WeakSet(); + +function exactRequiredBoundaries(boundaries: readonly LabSandboxBoundary[]): boolean { + const actual = new Set(boundaries); + return actual.size === REQUIRED_LAB_SANDBOX_BOUNDARIES.length + && REQUIRED_LAB_SANDBOX_BOUNDARIES.every((boundary) => actual.has(boundary)); +} + +/** + * Trusted non-Lab construction boundary for the exact-route executor. + * + * Callers may create this capability only after wiring enforcement for every required + * sandbox boundary. Missing or partial enforcement is rejected rather than silently + * producing live evidence with weaker guarantees. + */ +export function createTrustedLabRouteExecutor( + execute: LabRouteExecutor, + enforcedBoundaries: readonly LabSandboxBoundary[], +): TrustedLabRouteExecutor { + if (!exactRequiredBoundaries(enforcedBoundaries)) { + throw new Error("harness_failure: trusted Lab route executor lacks required sandbox enforcement"); + } + const capability: TrustedLabRouteExecutor = Object.freeze({ + execute, + enforcedBoundaries: Object.freeze([...enforcedBoundaries]), + }); + TRUSTED_EXECUTORS.add(capability); + return capability; +} + +export function isTrustedLabRouteExecutor(value: unknown): value is TrustedLabRouteExecutor { + return typeof value === "object" && value !== null && TRUSTED_EXECUTORS.has(value as object) + && exactRequiredBoundaries((value as TrustedLabRouteExecutor).enforcedBoundaries); +} diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts index a111d5f6e..0450236d2 100644 --- a/tests/lab-live-probe.test.ts +++ b/tests/lab-live-probe.test.ts @@ -9,10 +9,11 @@ import { registerMcpStubTool, routeSubjectApplicableToRequirements, runLiveScenario, runLiveSuite, scenarioManifestDigest, subjectIdForSubject, } from "../src/lab"; +import { createTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; import { replayLabLedger } from "../src/lab/ledger/store"; import { clearMcpStub } from "../src/lab/live/mcp-loopback"; import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; -import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext } from "../src/lab/live/types"; import type { RouteSubjectV1 } from "../src/lab/events/types"; import type { NormalizedObservation } from "../src/lab/conformance/types"; @@ -47,16 +48,24 @@ function transportForCase(caseRecord: ReturnType[n return createMockTransport({ entries: caseRecord.initiatingRequest ? [{ status: 200, headers: { "content-type": caseRecord.fixture.mediaType }, body: caseRecord.fixture.bytesUtf8 }] : [] }); } -function reasoningObservation(): NormalizedObservation { +function passObservation(): NormalizedObservation { return { - client: { request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "" } }, - upstream: { requests: [ - { status: 200, headers: {}, json: {}, rawBytes: 0 }, - { status: 200, headers: {}, json: { input: [{ signature: "sig_fixture" }] }, rawBytes: 0 }, - ], responses: [] }, - process: { exitCode: null }, verifiers: {}, + client: { request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "OK" } }, + upstream: { requests: [], responses: [] }, process: { exitCode: null }, verifiers: {}, }; } +function reasoningObservation(): NormalizedObservation { + const observation = passObservation(); + observation.upstream.requests = [ + { status: 200, headers: {}, json: {}, rawBytes: 0 }, + { status: 200, headers: {}, json: { input: [{ signature: "sig_fixture" }] }, rawBytes: 0 }, + ]; + observation.client.response.normalizedText = ""; + return observation; +} +function trustedObservation(observation: NormalizedObservation) { + return createTrustedLabRouteExecutor(async () => observation, REQUIRED_LAB_SANDBOX_BOUNDARIES); +} describe("CL-03 live probe harness", () => { test("loads frozen 10-case live authority with reproducible digests", () => { @@ -71,23 +80,31 @@ describe("CL-03 live probe harness", () => { test("runs only applicable live suite scenarios with an injected test transport", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const summary = await runLiveSuite(mockRoute(), ["responses-core", "chat-core"], { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!.fixture.bytesUtf8 }] }) }); - expect(summary.total).toBe(1); expect(summary.results[0]?.routeSubject?.subjectKind).toBe("route"); + expect(summary.total).toBe(1); expect(summary.results[0]?.routeSubject?.subjectKind).toBe("route"); expect(summary.results[0]?.executionAuthority).toBe("test_transport"); }); test("inert tool round-trips validate args and return static results", () => { expect(listInertToolDefinitions().map((t) => t.name)).toContain("lookup"); expect(executeInertTool("lookup", { q: "x" }).output).toBe("RESULT"); expect(() => executeInertTool("lookup", { q: 1 })).toThrow(); }); test("MCP lab stub only", () => { registerMcpStubTool({ namespace: "lab", name: "echo" }); expect(invokeMcpStub("lab", "echo", { x: 1 }, { content: [{ type: "text", text: "ECHO" }] }).content[0]?.text).toBe("ECHO"); }); - test("raw URLs and secrets never appear in persisted live evidence", async () => { + test("test transports can exercise normalization but cannot create live evidence", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); + expect(result.executionAuthority).toBe("test_transport"); + expect(() => observationFromLiveResult(result, caseRecord, authority, { configDir: home })).toThrow("trusted-route"); + }); + + test("raw URLs and secrets never appear in persisted trusted live evidence", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); + expect(result.executionAuthority).toBe("trusted_route"); const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); const serialized = JSON.stringify(event); expect(serialized).not.toContain("api.example.com"); expect(serialized).not.toContain("https://"); expect(serialized).not.toContain("apiKey"); expect(event.evidenceLayer).toBe("live_route_compatibility"); expect(event.executionMode).toBe("live"); }); - test("JSONL persistence and SQLite rebuild for live evidence", async () => { + test("JSONL persistence and SQLite rebuild for trusted live evidence", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!; const route = mockRoute({ effectiveAdapter: "openai-chat", upstreamProtocol: "openai-chat", surface: "responses-sse" }); - const result = await runLiveScenario(caseRecord, route, { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); + const result = await runLiveScenario(caseRecord, route, { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); persistLiveResult(result, caseRecord, authority, { configDir: home }); const projection = rebuildLabProjection(home); expect(projection.events).toBeGreaterThan(0); expect(projection.corruptions).toHaveLength(0); expect(replayLabLedger(join(home, "lab", "compatibility.jsonl")).events.length).toBeGreaterThan(0); }); @@ -118,7 +135,7 @@ describe("CL-03 live probe harness", () => { test("reasoning replay persists no private reasoning material", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const scenario = authority.cases.find((c) => c.id === "reasoning-core.live.replay")!; - const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: async () => reasoningObservation() }); + const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(reasoningObservation()) }); expect(result.passed).toBe(true); const { event } = observationFromLiveResult(result, scenario, authority, { configDir: home }); expect(JSON.stringify(event)).not.toContain("PLAN"); }); }); diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index 8a35d3ada..dd5f0ad55 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -7,7 +7,8 @@ import { createCredentialLease, createLabDestination, createMockTransport, createSandboxResourceState, enforceSandboxLimits, freezeRouteSubject, LabCredentialError, LabDestinationError, LabSandboxError, prepareLiveSandbox, rejectProxyEnvironment, TransportError, } from "../src/lab"; -import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; +import { createTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext } from "../src/lab/live/types"; const HOMES: string[] = []; function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } @@ -42,7 +43,14 @@ describe("CL-03 live sandbox security seams", () => { test("credential lease is destination-bound, bounded, and non-serializable", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const lease = createCredentialLease({ destination, budget: 1 }); assertLeaseScope(lease, destination); expect(() => JSON.stringify(lease)).toThrow(LabCredentialError); lease.consume(); expect(() => lease.consume()).toThrow(LabCredentialError); }); test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); - test("resource ceilings enforce child-process and memory limits", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); const state2 = createSandboxResourceState(); expect(() => enforceSandboxLimits(state2, { ...limits, maxMemoryBytes: 1 })).toThrow(LabSandboxError); }); + test("observable resource counters enforce child-process limit", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); }); + test("trusted executor cannot be constructed without the hard memory/process/time boundaries", () => { + const missingMemory = REQUIRED_LAB_SANDBOX_BOUNDARIES.filter((boundary) => boundary !== "memory_limit"); + expect(() => createTrustedLabRouteExecutor(async () => { throw new Error("never"); }, missingMemory)).toThrow("required sandbox enforcement"); + expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("memory_limit"); + expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("child_process_limit"); + expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("connect_timeout"); + }); test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); test("route subject stability across credential rotation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); createCredentialLease({ destination, budget: 1 }).consume(); const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); expect(subjectA).toEqual(subjectB); }); From ffc995822638a7d4bb2594b1597b6f965296a209 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:20:46 +0200 Subject: [PATCH 07/47] fix(lab): separate test execution from live evidence From e064e4ccdf6c2913cb68a13ef3436f69a808bc75 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:49:03 +0200 Subject: [PATCH 08/47] fix(lab): make installation salt creation atomic --- src/lab/subject/installation-salt.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts index c2d5654d2..df219889d 100644 --- a/src/lab/subject/installation-salt.ts +++ b/src/lab/subject/installation-salt.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; import { labInstallationSaltPath, labRoot } from "../paths"; @@ -7,15 +7,21 @@ const SALT_BYTES = 32; /** Read or create the per-installation salt used for local fingerprinting. */ export function readInstallationSalt(configDir?: string): Uint8Array { const path = labInstallationSaltPath(configDir); - if (existsSync(path)) { + const readExisting = (): Uint8Array => { const bytes = readFileSync(path); if (bytes.byteLength !== SALT_BYTES) { throw new Error("harness_failure: invalid installation salt length"); } return new Uint8Array(bytes); - } + }; + mkdirSync(labRoot(configDir), { recursive: true, mode: 0o700 }); const salt = randomBytes(SALT_BYTES); - writeFileSync(path, salt, { mode: 0o600 }); - return new Uint8Array(salt); + try { + writeFileSync(path, salt, { mode: 0o600, flag: "wx" }); + return new Uint8Array(salt); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return readExisting(); + } } From 30f2df00ef524a45c343c046f555c30e1cfd4574 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:49:36 +0200 Subject: [PATCH 09/47] fix(lab): sanitize destination policy failures --- src/lab/live/destination.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts index 6f1bfd723..4a3f4a994 100644 --- a/src/lab/live/destination.ts +++ b/src/lab/live/destination.ts @@ -32,7 +32,7 @@ function validateResolvedAddress( const assessment = assessUrlDestination(addressUrl(scheme, address.address)); if (!assessment) throw new LabDestinationError("resolver address could not be classified", "harness_failure"); if (["metadata", "link-local", "unspecified"].includes(assessment.kind)) { - throw new LabDestinationError(`blocked resolved destination: ${assessment.detail}`, "harness_failure"); + throw new LabDestinationError("blocked resolved destination", "harness_failure"); } const privateAnswer = ["private", "loopback", "localhost"].includes(assessment.kind); if (privateAnswer && (!allowPrivateNetwork || !labRunApproval)) { @@ -41,12 +41,16 @@ function validateResolvedAddress( return privateAnswer; } -function canonicalAddresses(addresses: Array<{ address: string; family: 4 | 6 }>): Array<{ address: string; family: 4 | 6 }> { +function canonicalAddresses(addresses: ReadonlyArray<{ address: string; family: 4 | 6 }>): Array<{ address: string; family: 4 | 6 }> { const unique = new Map(); for (const row of addresses) unique.set(`${row.family}:${row.address}`, { ...row }); return [...unique.values()].sort((a, b) => a.family - b.family || (a.address < b.address ? -1 : a.address > b.address ? 1 : 0)); } +function addressSetKey(addresses: ReadonlyArray<{ address: string; family: 4 | 6 }>): string { + return canonicalAddresses(addresses).map((row) => `${row.family}:${row.address}`).join(","); +} + export interface CreateDestinationOptions { baseUrl: string; allowPrivateNetwork?: boolean; @@ -66,7 +70,7 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom const scheme = parsed.protocol === "https:" ? "https" as const : "http" as const; const literal = assessUrlDestination(parsed.toString()); if (literal && ["metadata", "link-local", "unspecified"].includes(literal.kind)) { - throw new LabDestinationError(`blocked destination: ${literal.detail}`, "harness_failure"); + throw new LabDestinationError("blocked destination", "harness_failure"); } const allowPrivate = opts.allowPrivateNetwork === true; const approved = opts.labRunApproval === true; @@ -90,8 +94,8 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom }); addresses = resolved.addresses.map((row) => ({ address: row.address, family: row.family === 6 ? 6 as const : 4 as const })); privateNetwork = resolved.privateNetwork || privateNetwork; - } catch (error) { - throw new LabDestinationError(error instanceof Error ? error.message : "DNS destination policy failed", "network_blocked"); + } catch { + throw new LabDestinationError("DNS destination policy failed", "network_blocked"); } } const frozenAddresses = Object.freeze(canonicalAddresses(addresses).map((row) => Object.freeze(row))); @@ -115,9 +119,9 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom /** Detect any attempted replacement/re-resolution of the approved immutable address set. */ export function assertDestinationAddressSet(destination: LabDestinationV1, addresses: Array<{ address: string; family: 4 | 6 }>): void { - const a = destination.addresses.map((x) => `${x.family}:${x.address}`).sort().join(","); - const b = canonicalAddresses(addresses).map((x) => `${x.family}:${x.address}`).join(","); - if (a !== b) throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); + if (addressSetKey(destination.addresses) !== addressSetKey(addresses)) { + throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); + } } export function assertHostSniMatch(destination: LabDestinationV1, host: string, sni?: string): void { From 25cc54b4def13a2d3ee1e0970dd5452c279065ff Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:58:34 +0200 Subject: [PATCH 10/47] fix(lab): close live result classification type --- src/lab/live/types.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lab/live/types.ts b/src/lab/live/types.ts index d00b7c6fb..8a30cfc79 100644 --- a/src/lab/live/types.ts +++ b/src/lab/live/types.ts @@ -1,4 +1,4 @@ -import type { NormalizedObservation } from "../conformance/types"; +import type { FailureClassification, NormalizedObservation } from "../conformance/types"; import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types"; export interface LabDestinationV1 { @@ -152,7 +152,7 @@ export interface LabRouteExecutorInput { environment: Readonly>; } -/** Trusted exact-route execution function. The wrapper capability is branded outside Lab. */ +/** Trusted exact-route execution function. The wrapper capability is issued only by the host integration. */ export type LabRouteExecutor = (input: LabRouteExecutorInput) => Promise; export const REQUIRED_LAB_SANDBOX_BOUNDARIES = [ @@ -173,7 +173,7 @@ export const REQUIRED_LAB_SANDBOX_BOUNDARIES = [ ] as const; export type LabSandboxBoundary = typeof REQUIRED_LAB_SANDBOX_BOUNDARIES[number]; -/** Opaque-ish internal capability; runtime trust is established by a WeakSet in src/lib. */ +/** Opaque host-issued capability; runtime trust is established outside Lab. */ export interface TrustedLabRouteExecutor { execute(input: LabRouteExecutorInput): Promise; readonly enforcedBoundaries: readonly LabSandboxBoundary[]; @@ -187,7 +187,7 @@ export interface LiveScenarioRunResult { startedAt: number; completedAt: number; passed: boolean; - classification: string; + classification: FailureClassification; secondaryCode?: string; assertionResults: import("../conformance/types").AssertionResult[]; diagnostics: string[]; From 9b466132af05c6cda1353dc579208151b5039780 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:59:09 +0200 Subject: [PATCH 11/47] fix(lab): enforce pinned connect timeout --- src/lib/pinned-http.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/lib/pinned-http.ts b/src/lib/pinned-http.ts index 0b7123e4a..09b361fda 100644 --- a/src/lib/pinned-http.ts +++ b/src/lib/pinned-http.ts @@ -1,4 +1,4 @@ -import http, { type IncomingMessage, type RequestOptions } from "node:http"; +import http, { type ClientRequest, type IncomingMessage, type RequestOptions } from "node:http"; import https from "node:https"; export type PinnedAddress = { address: string; family: number }; @@ -6,6 +6,8 @@ export type PinnedAddress = { address: string; family: number }; export interface PinnedHttpRequestOptions { headers?: HeadersInit; maxBytes?: number; + /** Deadline for establishing the TCP connection and, for HTTPS, completing TLS. */ + connectTimeoutMs?: number; idleTimeoutMs?: number; rejectUnauthorized?: boolean; context?: string; @@ -27,6 +29,7 @@ function pinnedHttpRequest( throw new Error(`${options?.context ?? "request"} must use HTTP or HTTPS, got ${parsed.protocol}`); } const context = options?.context ?? "request"; + const connectTimeoutMs = options?.connectTimeoutMs ?? 10_000; const idleTimeoutMs = options?.idleTimeoutMs ?? 60_000; const maxBytes = options?.maxBytes; const headers = new Headers(options?.headers); @@ -44,8 +47,15 @@ function pinnedHttpRequest( } let settled = false; + let req: ClientRequest | undefined; + let connectTimer: ReturnType | undefined; + const clearConnectTimer = () => { + if (connectTimer !== undefined) clearTimeout(connectTimer); + connectTimer = undefined; + }; const fail = (error: unknown) => { - try { req.destroy(); } catch { /* ignore */ } + clearConnectTimer(); + try { req?.destroy(); } catch { /* ignore */ } if (settled) return; settled = true; reject(error instanceof Error ? error : new Error(String(error))); @@ -83,6 +93,7 @@ function pinnedHttpRequest( }; const onResponse = (response: IncomingMessage) => { + clearConnectTimer(); const status = response.statusCode ?? 0; const responseHeaders = new Headers(); for (const [key, value] of Object.entries(response.headers)) { @@ -96,7 +107,7 @@ function pinnedHttpRequest( if (status < 200 || status >= 300) { try { response.destroy(); } catch { /* ignore */ } - try { req.destroy(); } catch { /* ignore */ } + try { req?.destroy(); } catch { /* ignore */ } if (settled) return; settled = true; resolve(new Response(null, { status, headers: responseHeaders })); @@ -131,7 +142,7 @@ function pinnedHttpRequest( }); }, cancel() { - req.destroy(); + req?.destroy(); }, }); @@ -141,15 +152,26 @@ function pinnedHttpRequest( }; const requestFn = parsed.protocol === "https:" ? https.request : http.request; - const req = requestFn(requestOptions, onResponse); + req = requestFn(requestOptions, onResponse); const onAbort = () => fail(signal?.reason instanceof Error ? signal.reason : new Error("aborted")); signal?.addEventListener("abort", onAbort, { once: true }); + req.on("socket", (socket) => { + if (!socket.connecting) return; + const connectedEvent = parsed.protocol === "https:" ? "secureConnect" : "connect"; + connectTimer = setTimeout(() => fail(new Error(`${context} connect timed out`)), connectTimeoutMs); + socket.once(connectedEvent, clearConnectTimer); + socket.once("error", clearConnectTimer); + socket.once("close", clearConnectTimer); + }); req.setTimeout(idleTimeoutMs, () => fail(new Error(`${context} timed out`))); req.on("error", error => { signal?.removeEventListener("abort", onAbort); fail(error); }); - req.on("close", () => signal?.removeEventListener("abort", onAbort)); + req.on("close", () => { + clearConnectTimer(); + signal?.removeEventListener("abort", onAbort); + }); req.end(body); }); } From 735beb2b3d61950421786497fbaedc26f3a90f9d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:59:32 +0200 Subject: [PATCH 12/47] fix(lab): constrain live response metadata --- src/lib/lab-live-pinned-sender.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/lab-live-pinned-sender.ts b/src/lib/lab-live-pinned-sender.ts index 56c1e2e5b..980bb6352 100644 --- a/src/lib/lab-live-pinned-sender.ts +++ b/src/lib/lab-live-pinned-sender.ts @@ -1,6 +1,9 @@ import { pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; import type { LabCredentialLeaseV1, LabPinnedSender } from "../lab/live/types"; +/** Only response metadata required by current live assertions crosses into Lab. */ +const LAB_RESPONSE_HEADER_ALLOWLIST = ["content-type"] as const; + /** * Trusted credential/transport owner. Secret headers exist only in this non-Lab module and are * consumed directly by the pinned HTTP primitive; they are never returned to Lab code. @@ -14,6 +17,7 @@ export function createLabAuthorizedPinnedSender( const options = { headers, maxBytes: limits.maxOutputBytes, + connectTimeoutMs: limits.connectTimeoutMs, idleTimeoutMs: Math.min(limits.firstByteTimeoutMs, limits.inactivityTimeoutMs), rejectUnauthorized: true, context: "Lab provider response", @@ -23,7 +27,10 @@ export function createLabAuthorizedPinnedSender( : await pinnedHttpGet(url, pinned, signal, options); const body = await response.text(); const responseHeaders: Record = {}; - response.headers.forEach((value, key) => { responseHeaders[key.toLowerCase()] = value; }); + for (const headerName of LAB_RESPONSE_HEADER_ALLOWLIST) { + const value = response.headers.get(headerName); + if (value !== null) responseHeaders[headerName] = value; + } return { status: response.status, headers: responseHeaders, body }; }; } From 98bfcf23ed1f652a4bede1a9a126ec82cbad80c0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:00:08 +0200 Subject: [PATCH 13/47] fix(lab): type live failure classification --- src/lab/live/transport.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index bc1ced001..90b0b8bea 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -1,3 +1,4 @@ +import type { FailureClassification } from "../conformance/types"; import { assertLeaseScope, LabCredentialError } from "./credential-lease"; import type { LabCredentialLeaseV1, LabDestinationV1, LabPinnedSender, LabTransport, LabTransportRequest, @@ -95,7 +96,7 @@ export class TransportError extends Error { constructor(readonly code: TransportErrorCode, message: string) { super(message); } } -export function classifyTransportError(error: unknown): { classification: string; secondaryCode: string } { +export function classifyTransportError(error: unknown): { classification: FailureClassification; secondaryCode: string } { const code = error instanceof TransportError ? error.code : undefined; switch (code) { case "auth_blocked": return { classification: "authentication_blocked", secondaryCode: code }; From 836be5695de2481a04c5b4b40478e04dd6cc7425 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:02:51 +0200 Subject: [PATCH 14/47] fix(lab): bind trusted live results to execution receipts --- src/lab/live/executor.ts | 107 ++++++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 23 deletions(-) diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index 4b31f07c4..5d1ae89dd 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -1,18 +1,20 @@ import { evaluateAssertions } from "../conformance/assertion"; import { emptyObservation, finalizeObservation, setClientResponse } from "../conformance/observation"; import { normalizeSseBytes } from "../conformance/sse-normalize"; -import type { CaseRecord, FailureClassification, FailureRule, NormalizedEvent, NormalizedObservation } from "../conformance/types"; +import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule, NormalizedEvent, NormalizedObservation } from "../conformance/types"; +import { domainHash, jcsStringify, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; import type { RouteSubjectV1 } from "../events/types"; import { isTrustedLabRouteExecutor } from "../../lib/lab-live-execution-authority"; import { buildRouteSubjectV1 } from "../subject/route-subject"; import { createLabDestination, LabDestinationError } from "./destination"; -import { expandLiveScenario } from "./manifest"; +import { expandLiveScenario, loadLiveCaseAuthority } from "./manifest"; import { createSandboxResourceState, enforceSandboxLimits, LabSandboxError, prepareLiveSandbox } from "./sandbox"; +import { liveSuiteManifestObjectForCase } from "./suite-manifest"; import { classifyTransportError, TransportError } from "./transport"; import type { DnsResolver, LabRouteContext, LabTransport, LiveExecutionAuthority, LiveRunConfig, LiveScenarioRunResult, TrustedLabRouteExecutor } from "./types"; export interface LiveExecutorOptions { - /** Branded exact-route capability for evidence-eligible production execution. */ + /** Host-issued exact-route capability for evidence-eligible production execution. */ routeExecutor?: TrustedLabRouteExecutor; /** Test-only byte transport. Results are deliberately not evidence-eligible. */ transport?: LabTransport; @@ -21,6 +23,49 @@ export interface LiveExecutorOptions { env?: NodeJS.ProcessEnv; } +interface TrustedLiveResultReceipt { + authorityDigest: string; + scenarioId: string; + suiteId: string; + scenarioManifestDigest: string; + suiteManifestDigest: string; + routeSubjectId: string; +} + +const TRUSTED_RESULT_RECEIPTS = new WeakMap(); +const LIVE_AUTHORITY_DOMAIN = "ocx-lab:live-authority:v1"; + +function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): TrustedLiveResultReceipt { + if (!result.routeSubject) throw new Error("trusted live result has no route subject"); + return { + authorityDigest: domainHash(LIVE_AUTHORITY_DOMAIN, jcsStringify(authority)), + scenarioId: caseRecord.id, + suiteId: caseRecord.suite, + scenarioManifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), + suiteManifestDigest: suiteManifestDigest(liveSuiteManifestObjectForCase(caseRecord, authority)), + routeSubjectId: subjectIdForSubject(result.routeSubject), + }; +} + +function sealTrustedLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): void { + TRUSTED_RESULT_RECEIPTS.set(result, receiptFor(result, caseRecord, authority)); +} + +/** Verify the module-private execution receipt before any live result enters persistence. */ +export function assertTrustedLiveResultReceipt(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): void { + const actual = TRUSTED_RESULT_RECEIPTS.get(result); + if (!actual || result.executionAuthority !== "trusted_route") { + throw new Error("live result lacks trusted execution receipt"); + } + const expected = receiptFor(result, caseRecord, authority); + for (const key of Object.keys(expected) as Array) { + if (actual[key] !== expected[key]) throw new Error("live result trusted execution receipt mismatch"); + } + if (result.scenarioId !== caseRecord.id || result.suite !== caseRecord.suite) { + throw new Error("live result scenario identity mismatch"); + } +} + function liveLimitsFromAuthority(authorityLimits: Record): LiveRunConfig { return { totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, @@ -150,30 +195,43 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR const startedAt = Date.now(); let routeSubject: RouteSubjectV1 | undefined; let executionAuthority: LiveExecutionAuthority = "none"; - const complete = (partial: Omit): LiveScenarioRunResult => ({ ...partial, executionAuthority, startedAt, completedAt: Math.max(startedAt, Date.now()) }); + let authority: CaseAuthority | undefined; + let activeCase = caseRecord; + let trustedExecutionStarted = false; + const complete = (partial: Omit): LiveScenarioRunResult => { + const result: LiveScenarioRunResult = { ...partial, executionAuthority, startedAt, completedAt: Math.max(startedAt, Date.now()) }; + if (trustedExecutionStarted && authority && result.routeSubject) sealTrustedLiveResult(result, activeCase, authority); + return result; + }; try { const environment = prepareLiveSandbox(opts.env); - if (!isLiveCaseApplicableToRoute(caseRecord, routeContext)) { - return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: "inconclusive", secondaryCode: "scenario_inapplicable", assertionResults: [], diagnostics, routeSubject }); + authority = loadLiveCaseAuthority(); + const canonicalCase = authority.cases.find((row) => row.id === caseRecord.id); + if (!canonicalCase || jcsStringify(canonicalCase) !== jcsStringify(caseRecord)) { + throw new TransportError("harness_failure", "live scenario does not match canonical authority"); } - const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); - routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); - const preconditionFailure = routePreconditionFailure(routeContext, caseRecord); + activeCase = canonicalCase; + if (!isLiveCaseApplicableToRoute(activeCase, routeContext)) { + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: "inconclusive", secondaryCode: "scenario_inapplicable", assertionResults: [], diagnostics, routeSubject }); + } + const preconditionFailure = routePreconditionFailure(routeContext, activeCase); if (preconditionFailure) { - return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: "inconclusive", secondaryCode: preconditionFailure, assertionResults: [], diagnostics: [preconditionFailure], routeSubject }); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: "inconclusive", secondaryCode: preconditionFailure, assertionResults: [], diagnostics: [preconditionFailure], routeSubject }); } - const authority = await import("./manifest").then((m) => m.loadLiveCaseAuthority()); - const expanded = expandLiveScenario(caseRecord, authority); + const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); + routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); + const expanded = expandLiveScenario(activeCase, authority); const limits = liveLimitsFromAuthority(expanded.executionLimits as Record); const state = createSandboxResourceState(); let observation: NormalizedObservation; if (opts.routeExecutor) { if (!isTrustedLabRouteExecutor(opts.routeExecutor)) throw new TransportError("untrusted_route_executor", "untrusted route executor capability"); executionAuthority = "trusted_route"; - observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: caseRecord.id, initiatingRequest: caseRecord.initiatingRequest?.bytesUtf8, limits, signal, environment })); - } else if (opts.transport && caseRecord.initiatingRequest) { + trustedExecutionStarted = true; + observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: activeCase.id, initiatingRequest: activeCase.initiatingRequest?.bytesUtf8, limits, signal, environment })); + } else if (opts.transport && activeCase.initiatingRequest) { executionAuthority = "test_transport"; - const body = caseRecord.initiatingRequest.bytesUtf8; + const body = activeCase.initiatingRequest.bytesUtf8; const inputBytes = new TextEncoder().encode(body).byteLength; enforceSandboxLimits(state, limits, { requests: 1, inputBytes }); const response = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.transport!.request({ method: "POST", path: pathForProtocol(routeContext.upstreamProtocol), body, signal })); @@ -185,23 +243,26 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR enforceSandboxLimits(state, limits, { outputBytes, outputTokens: Math.ceil(outputBytes / 4) }); observation = normalizeTransportObservation(routeContext, response); } else { - throw new TransportError("live_transport_required", caseRecord.initiatingRequest ? "live transport or trusted route executor required" : "trusted route executor required"); + throw new TransportError("live_transport_required", activeCase.initiatingRequest ? "live transport or trusted route executor required" : "trusted route executor required"); } - const assertionResults = evaluateAssertions(caseRecord.assertions, observation); + const assertionResults = evaluateAssertions(activeCase.assertions, observation); const requiredFailures = assertionResults.filter((row) => row.required && !row.passed); const failureRules = expanded.failureRules as FailureRule[]; - if (caseRecord.expectedFailure) { - const matched = caseRecord.expectedFailure.assertionIds.every((id) => assertionResults.find((row) => row.id === id)?.passed === true) && requiredFailures.length === 0; - return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: matched, classification: matched ? caseRecord.expectedFailure.expectedClass : "protocol_failure", secondaryCode: matched ? caseRecord.expectedFailure.expectedCode : "deterministic_assertion", assertionResults, diagnostics, routeSubject }); + if (activeCase.expectedFailure) { + const matched = activeCase.expectedFailure.assertionIds.every((id) => assertionResults.find((row) => row.id === id)?.passed === true) && requiredFailures.length === 0; + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: matched, classification: matched ? activeCase.expectedFailure.expectedClass : "protocol_failure", secondaryCode: matched ? activeCase.expectedFailure.expectedCode : "deterministic_assertion", assertionResults, diagnostics, routeSubject }); } const passed = requiredFailures.length === 0; const classified = passed ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass" } : classifyWithFailureRules(failureRules, "required_assertion_failed"); - return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }); } catch (error) { - diagnostics.push(error instanceof Error ? `${error.name}:${error.message}` : String(error)); + const diagnosticCode = error instanceof TransportError || error instanceof LabSandboxError || error instanceof LabDestinationError + ? error.code + : "execution_error"; + diagnostics.push(diagnosticCode); let classified = classifyTransportError(error); if (error instanceof LabSandboxError) classified = { classification: error.code === "harness_failure" ? "harness_failure" : "budget_exhausted", secondaryCode: error.code }; if (error instanceof LabDestinationError) classified = { classification: error.code === "network_blocked" ? "network_failure" : "harness_failure", secondaryCode: error.code }; - return complete({ scenarioId: caseRecord.id, suite: caseRecord.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }); } } From c05df3cf18b27400e1f07945422698ba22dd3701 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:03:27 +0200 Subject: [PATCH 15/47] fix(lab): require trusted receipt before live persistence --- src/lab/observe/from-live.ts | 48 +++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index ace436bd4..a34809024 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -2,11 +2,12 @@ import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PRODUCER_VERSION, OBSERVATION_LIMIT_NAMES, type ObservationOutcome } from "../constants"; import { fixtureDigest, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; -import type { ObservationEvent } from "../events/types"; +import type { FailureRecordV1, ObservationEvent } from "../events/types"; import { assignEventId } from "../events/validate"; import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import type { CaseAuthority, CaseRecord } from "../conformance/types"; +import { assertTrustedLiveResultReceipt } from "../live/executor"; import { expandLiveScenario } from "../live/manifest"; import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; import type { LiveScenarioRunResult } from "../live/types"; @@ -16,10 +17,38 @@ export interface PersistedLiveObservation { event: ObservationEvent; ledgerPath: function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; - if (["timeout", "budget_exhausted", "authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient"].includes(result.classification)) return "blocked"; - if (["inconclusive", "harness_failure"].includes(result.classification)) return "inconclusive"; - if (["protocol_failure", "capability_failure", "behavioral_failure"].includes(result.classification)) return "fail"; - return "inconclusive"; + switch (result.classification) { + case "timeout": + case "budget_exhausted": + case "authentication_blocked": + case "quota_blocked": + case "region_blocked": + case "network_failure": + case "provider_transient": + return "blocked"; + case "protocol_failure": + case "capability_failure": + case "behavioral_failure": + return "fail"; + case "inconclusive": + case "harness_failure": + return "inconclusive"; + } +} + +function failureFromLiveResult(result: LiveScenarioRunResult): FailureRecordV1 | undefined { + if (result.passed || result.classification === "inconclusive") return undefined; + const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" + ? "harness" + : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) + ? "environment" + : "route"; + return { + class: result.classification, + code: result.secondaryCode ?? result.classification, + retryable: false, + attribution, + }; } function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveOptions): { startedAt: number; completedAt: number } { @@ -29,11 +58,12 @@ function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveO } export function observationFromLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): { event: ObservationEvent; artifacts: ReturnType[] } { - if (result.executionAuthority !== "trusted_route") throw new Error("only trusted-route execution may be persisted as live evidence"); + assertTrustedLiveResultReceipt(result, caseRecord, authority); if (!result.routeSubject) throw new Error("live evidence without an exact RouteSubjectV1 is not persistable"); + const { startedAt, completedAt } = requireExecutionTimes(result, opts); const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const { startedAt, completedAt } = requireExecutionTimes(result, opts); const recordedAt = opts.recordedAt ?? completedAt; + const recordedAt = opts.recordedAt ?? completedAt; const expandedScenario = expandLiveScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); const suiteExpanded = liveSuiteManifestObjectForCase(caseRecord, authority); const suiteDigest = suiteManifestDigest(suiteExpanded); const fixtureDigests: string[] = []; const artifacts: ReturnType[] = []; @@ -48,6 +78,7 @@ export function observationFromLiveResult(result: LiveScenarioRunResult, caseRec assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, observedSummary: row.observedSummary, reason: row.reason })) } })); const subject = result.routeSubject; const subjectId = subjectIdForSubject(subject); const authorityLimits = authority.manifestDefaults.executionLimits; const limits: Record = {}; for (const key of OBSERVATION_LIMIT_NAMES) if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; + const failure = failureFromLiveResult(result); const eventWithoutId = { schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, evidenceLayer: "live_route_compatibility" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), scenarioManifestDigest: scenarioDigest, @@ -56,8 +87,7 @@ export function observationFromLiveResult(result: LiveScenarioRunResult, caseRec assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, expectedSummary: "see_assertion_report", observedSummary: row.observedSummary.slice(0, 512), ...(row.reason ? { reason: row.reason } : {}) })), ...(caseRecord.expectedFailure ? { expectedFailure: { ...caseRecord.expectedFailure } } : {}), environment: { runtime: { platform: process.platform, arch: process.arch, bunVersion: process.versions.bun ?? Bun.version } }, artifactRefs: artifacts, - ...(result.passed ? {} : { failure: { class: result.classification, code: result.secondaryCode ?? result.classification, retryable: false, - attribution: result.classification === "harness_failure" ? "harness" as const : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) ? "environment" as const : "route" as const } }), + ...(failure ? { failure } : {}), }; return { event: assignEventId(eventWithoutId) as ObservationEvent, artifacts }; } finally { if (ownsStore) store.close(); } From 316eee612ea2261085d4e882c2ccb6f8f528a883 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:03:51 +0200 Subject: [PATCH 16/47] fix(lab): enforce byte-identical live authority mirror --- src/lab/live/manifest.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lab/live/manifest.ts b/src/lab/live/manifest.ts index c78adc45e..c62bee96a 100644 --- a/src/lab/live/manifest.ts +++ b/src/lab/live/manifest.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { fixtureDigest, jcsStringify } from "../digest"; +import { fixtureDigest } from "../digest"; import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule } from "../conformance/types"; import { CL03_LIVE_SUITES, SYNTHETIC_MARKER } from "../conformance/types"; @@ -12,17 +12,18 @@ const REPO_AUTHORITY = join(MODULE_DIR, "..", "..", "..", "devlog", "_plan", "26 function readAuthority(path: string): CaseAuthority { return JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; } -/** During source-tree execution, fail closed if the packaged authority copy drifts from normative 024. */ -export function assertLiveAuthorityMirror(runtime: CaseAuthority): void { +/** During source-tree execution, fail closed unless the runtime copy is byte-identical to normative 024. */ +export function assertLiveAuthorityMirror(): void { if (!existsSync(REPO_AUTHORITY)) return; - const normative = readAuthority(REPO_AUTHORITY); - if (jcsStringify(runtime) !== jcsStringify(normative)) throw new Error("harness_failure: live authority runtime copy drift"); + const runtimeBytes = readFileSync(RUNTIME_AUTHORITY); + const normativeBytes = readFileSync(REPO_AUTHORITY); + if (!runtimeBytes.equals(normativeBytes)) throw new Error("harness_failure: live authority runtime copy drift"); } export function loadLiveCaseAuthority(): CaseAuthority { + assertLiveAuthorityMirror(); const raw = readAuthority(RUNTIME_AUTHORITY); validateLiveAuthority(raw); - assertLiveAuthorityMirror(raw); return raw; } From a175558514f883fe07ba0ea076f33e75e64d3688 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:05:20 +0200 Subject: [PATCH 17/47] fix(lab): correct live manifest authority limits --- src/lab/conformance/fixtures/live-v1-cases.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lab/conformance/fixtures/live-v1-cases.json b/src/lab/conformance/fixtures/live-v1-cases.json index ded71e84a..4301deeca 100644 --- a/src/lab/conformance/fixtures/live-v1-cases.json +++ b/src/lab/conformance/fixtures/live-v1-cases.json @@ -45,8 +45,7 @@ "maxChildProcesses": 0, "maxArtifacts": 16, "perArtifactBytes": 262144, - "aggregateArtifactBytes": 1048576, - "maxArtifactBytes": 262144 + "aggregateArtifactBytes": 1048576 }, "artifactPolicy": { "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], @@ -144,7 +143,7 @@ "id": "vision-core.live.synthetic-ocr", "suite": "vision-core", "capability": "modalities.image.input", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, "assertions": [ @@ -173,4 +172,4 @@ ] } ] -} +} \ No newline at end of file From 73b138131669dc014895d8eca184ee86dcb31bf4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:05:59 +0200 Subject: [PATCH 18/47] chore(lab): prepare authority mirror sync --- .../260807_compatibility_lab/.cl03-authority-sync-placeholder | 1 + 1 file changed, 1 insertion(+) create mode 100644 devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder diff --git a/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder b/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder new file mode 100644 index 000000000..dcdd4815d --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder @@ -0,0 +1 @@ +sync \ No newline at end of file From 69159b3a56aef171f12da358f11532f721ee6de9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:07:11 +0200 Subject: [PATCH 19/47] fix(lab): sync normative live authority bytes --- .../.cl03-authority-sync-placeholder | 1 - .../_plan/260807_compatibility_lab/024_live_v1_cases.json | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder diff --git a/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder b/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder deleted file mode 100644 index dcdd4815d..000000000 --- a/devlog/_plan/260807_compatibility_lab/.cl03-authority-sync-placeholder +++ /dev/null @@ -1 +0,0 @@ -sync \ No newline at end of file diff --git a/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json index ded71e84a..4301deeca 100644 --- a/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json +++ b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json @@ -45,8 +45,7 @@ "maxChildProcesses": 0, "maxArtifacts": 16, "perArtifactBytes": 262144, - "aggregateArtifactBytes": 1048576, - "maxArtifactBytes": 262144 + "aggregateArtifactBytes": 1048576 }, "artifactPolicy": { "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], @@ -144,7 +143,7 @@ "id": "vision-core.live.synthetic-ocr", "suite": "vision-core", "capability": "modalities.image.input", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, "assertions": [ @@ -173,4 +172,4 @@ ] } ] -} +} \ No newline at end of file From 6fd8d4a3d665bdabd488172df3d891ce33042183 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:08:54 +0200 Subject: [PATCH 20/47] fix(lab): fail closed on live projection applicability --- src/lab/projection/verification.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 1b7ea2e88..ac48e342d 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -52,17 +52,17 @@ function scenarioApplicableToRequirements( ); } -/** Applicability for live_route_compatibility using RouteSubjectV1 fields. */ +/** Applicability for live_route_compatibility using exact RouteSubjectV1 plus validated claim state. */ export function routeSubjectApplicableToRequirements( requirements: ScenarioRequirements, subject: RouteSubjectV1, - routeRequiredClaims: string[] = [], + supportedClaims: readonly string[], ): boolean { const inbound = requirements.inboundProtocols ?? []; const upstream = requirements.upstreamProtocols ?? []; const surfaces = requirements.surfaces ?? []; const requiredClaims = requirements.requiredClaims ?? []; - const claimsOk = requiredClaims.every((claim) => routeRequiredClaims.includes(claim)); + const claimsOk = requiredClaims.every((claim) => supportedClaims.includes(claim)); return ( inbound.includes(subject.inboundProtocol) && upstream.includes(subject.upstreamProtocol) && @@ -99,7 +99,8 @@ function scenarioContractFromManifest( const upstreamProtocols = parseStringArray(row.upstreamProtocols); const surfaces = parseStringArray(row.surfaces); if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; - const requiredClaims = parseStringArray(row.requiredClaims) ?? []; + const requiredClaims = row.requiredClaims === undefined ? [] : parseStringArray(row.requiredClaims); + if (!requiredClaims) return null; const freshness = parseFreshness(scenarioManifest.freshness); if (!freshness) return null; return { inboundProtocols, upstreamProtocols, surfaces, requiredClaims, freshness }; @@ -139,7 +140,8 @@ export function evaluateAllApplicableRequiredPassV1( executionMode: ExecutionMode, opts: { subject?: ProtocolSubjectV1 | RouteSubjectV1; - routeRequiredClaims?: string[]; + /** For live projection this must come from validated current claim snapshots for subjectId. */ + routeSupportedClaims?: readonly string[]; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; asOf?: number; @@ -155,6 +157,14 @@ export function evaluateAllApplicableRequiredPassV1( notes: ["unsupported_verification_rule"], }; } + if (suiteManifest.evidenceLayer === "live_route_compatibility") { + if (opts.subject?.subjectKind !== "route") { + return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_subject_required"] }; + } + if (opts.routeSupportedClaims === undefined) { + return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_claim_state_required"] }; + } + } const requiredScenarios = suiteManifest.scenarios.filter( (s) => s.role === "required" || s.role === "negative_control", @@ -186,7 +196,7 @@ export function evaluateAllApplicableRequiredPassV1( if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; } if (suiteManifest.evidenceLayer === "live_route_compatibility" && opts.subject?.subjectKind === "route") { - if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeRequiredClaims ?? [])) continue; + if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeSupportedClaims!)) continue; } applicableRequired.push(s.id); scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); From 6950e2131f7770985c6913315e896f954afe75e5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:09:39 +0200 Subject: [PATCH 21/47] fix(lab): derive live applicability from ledger claims --- src/lab/projection/verdicts.ts | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index cafe66cd6..db0bc683e 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -166,6 +166,16 @@ export function resolveClaimStates( return { states, corruptions }; } +function supportedClaimsForSubject(claimStates: Map, subjectId: string): string[] { + const supported = new Set(); + for (const state of claimStates.values()) { + const claim = state.current; + if (!claim || state.corruption || state.unusable) continue; + if (claim.subjectId === subjectId && claim.polarity === "supported") supported.add(claim.capability); + } + return [...supported].sort(); +} + /** * CL-02 verdict projection primitives with frozen CL-00 verification semantics. */ @@ -232,6 +242,9 @@ export function projectVerdicts( projectObservationGroup(key, ordered, asOf, suiteManifest, { loadScenarioManifest: opts.loadScenarioManifest, loadScenarioRequirements: opts.loadScenarioRequirements, + routeSupportedClaims: key.evidenceLayer === "live_route_compatibility" + ? supportedClaimsForSubject(claimStates, key.subjectId) + : undefined, }), ); } @@ -296,6 +309,7 @@ function projectObservationGroup( opts: { loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + routeSupportedClaims?: readonly string[]; } = {}, ): DerivedVerdict { const contributing: string[] = []; @@ -365,6 +379,34 @@ function projectObservationGroup( notes.push("incomplete_required_coverage"); } } + } else if (key.evidenceLayer === "live_route_compatibility" && newestCurrent?.executionMode === "live") { + if (!suiteManifest) { + verdict = "PROBED"; + notes.push("suite_manifest_unavailable"); + } else { + const evaluation = evaluateAllApplicableRequiredPassV1( + suiteManifest, + ordered, + newestCurrent.executionMode, + { + subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, + routeSupportedClaims: opts.routeSupportedClaims, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.applicableRequiredScenarioIds.length === 0) { + verdict = "UNKNOWN"; + } else if (evaluation.canVerify) { + verdict = "VERIFIED"; + notes.push("all-applicable-required-pass-v1"); + } else { + verdict = "PROBED"; + notes.push("incomplete_required_coverage"); + } + } } else { verdict = "PROBED"; } From 83c72db90c9e69d34dfd4a3305051eecead82234 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:12:37 +0200 Subject: [PATCH 22/47] fix(lab): remove forgeable executor authority factory --- src/lib/lab-live-execution-authority.ts | 42 +++++-------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/src/lib/lab-live-execution-authority.ts b/src/lib/lab-live-execution-authority.ts index 75b172d52..6bf0dbd34 100644 --- a/src/lib/lab-live-execution-authority.ts +++ b/src/lib/lab-live-execution-authority.ts @@ -1,41 +1,13 @@ -import type { - LabRouteExecutor, - LabSandboxBoundary, - TrustedLabRouteExecutor, -} from "../lab/live/types"; -import { REQUIRED_LAB_SANDBOX_BOUNDARIES } from "../lab/live/types"; - -const TRUSTED_EXECUTORS = new WeakSet(); - -function exactRequiredBoundaries(boundaries: readonly LabSandboxBoundary[]): boolean { - const actual = new Set(boundaries); - return actual.size === REQUIRED_LAB_SANDBOX_BOUNDARIES.length - && REQUIRED_LAB_SANDBOX_BOUNDARIES.every((boundary) => actual.has(boundary)); -} +import type { TrustedLabRouteExecutor } from "../lab/live/types"; +import { isHostIssuedLabRouteExecutor } from "./lab-live-host"; /** - * Trusted non-Lab construction boundary for the exact-route executor. + * Read-only trust check used by the Lab executor. * - * Callers may create this capability only after wiring enforcement for every required - * sandbox boundary. Missing or partial enforcement is rejected rather than silently - * producing live evidence with weaker guarantees. + * Capability issuance intentionally does not live in this authority module: callers cannot + * self-attest sandbox boundary names here. Only the dedicated trusted host integration may + * construct and register an evidence-eligible route executor. */ -export function createTrustedLabRouteExecutor( - execute: LabRouteExecutor, - enforcedBoundaries: readonly LabSandboxBoundary[], -): TrustedLabRouteExecutor { - if (!exactRequiredBoundaries(enforcedBoundaries)) { - throw new Error("harness_failure: trusted Lab route executor lacks required sandbox enforcement"); - } - const capability: TrustedLabRouteExecutor = Object.freeze({ - execute, - enforcedBoundaries: Object.freeze([...enforcedBoundaries]), - }); - TRUSTED_EXECUTORS.add(capability); - return capability; -} - export function isTrustedLabRouteExecutor(value: unknown): value is TrustedLabRouteExecutor { - return typeof value === "object" && value !== null && TRUSTED_EXECUTORS.has(value as object) - && exactRequiredBoundaries((value as TrustedLabRouteExecutor).enforcedBoundaries); + return isHostIssuedLabRouteExecutor(value); } From 6128f6fd0a45dba14076e5cfc645ff88f35e5978 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:12:53 +0200 Subject: [PATCH 23/47] fix(lab): issue live executor capabilities in host integration --- src/lib/lab-live-host.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/lib/lab-live-host.ts diff --git a/src/lib/lab-live-host.ts b/src/lib/lab-live-host.ts new file mode 100644 index 000000000..9808d1745 --- /dev/null +++ b/src/lib/lab-live-host.ts @@ -0,0 +1,30 @@ +import type { LabRouteExecutor, TrustedLabRouteExecutor } from "../lab/live/types"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES } from "../lab/live/types"; + +const HOST_ISSUED_EXECUTORS = new WeakSet(); + +/** + * Trusted host integration boundary for CL-03 exact-route execution. + * + * This module is intentionally not re-exported from the Lab public surface. The host owns + * enforcement of every mandatory sandbox boundary and therefore supplies no caller-controlled + * boundary list. Adapters/providers receive only the resulting opaque capability. + * + * @internal host integration only + */ +export function createHostIssuedLabRouteExecutor(execute: LabRouteExecutor): TrustedLabRouteExecutor { + const capability: TrustedLabRouteExecutor = Object.freeze({ + execute, + enforcedBoundaries: Object.freeze([...REQUIRED_LAB_SANDBOX_BOUNDARIES]), + }); + HOST_ISSUED_EXECUTORS.add(capability); + return capability; +} + +/** Internal recognition check consumed by the read-only authority facade. */ +export function isHostIssuedLabRouteExecutor(value: unknown): value is TrustedLabRouteExecutor { + if (typeof value !== "object" || value === null || !HOST_ISSUED_EXECUTORS.has(value as object)) return false; + const capability = value as TrustedLabRouteExecutor; + return capability.enforcedBoundaries.length === REQUIRED_LAB_SANDBOX_BOUNDARIES.length + && REQUIRED_LAB_SANDBOX_BOUNDARIES.every((boundary, index) => capability.enforcedBoundaries[index] === boundary); +} From b4e27981f58da13d6f40e7b78afd546929b85ddc Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:14:09 +0200 Subject: [PATCH 24/47] test(lab): cover trusted live evidence boundaries --- tests/lab-live-probe.test.ts | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts index 0450236d2..95d40e7f0 100644 --- a/tests/lab-live-probe.test.ts +++ b/tests/lab-live-probe.test.ts @@ -9,12 +9,12 @@ import { registerMcpStubTool, routeSubjectApplicableToRequirements, runLiveScenario, runLiveSuite, scenarioManifestDigest, subjectIdForSubject, } from "../src/lab"; -import { createTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; import { replayLabLedger } from "../src/lab/ledger/store"; import { clearMcpStub } from "../src/lab/live/mcp-loopback"; import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; -import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext } from "../src/lab/live/types"; -import type { RouteSubjectV1 } from "../src/lab/events/types"; +import type { LabBehaviorValues, LabRouteContext, LiveScenarioRunResult } from "../src/lab/live/types"; +import type { ProtocolSubjectV1, RouteSubjectV1 } from "../src/lab/events/types"; import type { NormalizedObservation } from "../src/lab/conformance/types"; const HOMES: string[] = []; @@ -64,7 +64,11 @@ function reasoningObservation(): NormalizedObservation { return observation; } function trustedObservation(observation: NormalizedObservation) { - return createTrustedLabRouteExecutor(async () => observation, REQUIRED_LAB_SANDBOX_BOUNDARIES); + return createHostIssuedLabRouteExecutor(async () => observation); +} + +function routeSubject(): RouteSubjectV1 { + return { subjectSchemaVersion: 1, subjectKind: "route", providerId: "p", providerInstanceFingerprint: "a".repeat(64), clientModelId: "m", upstreamModelId: "m", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; } describe("CL-03 live probe harness", () => { @@ -90,7 +94,32 @@ describe("CL-03 live probe harness", () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); expect(result.executionAuthority).toBe("test_transport"); - expect(() => observationFromLiveResult(result, caseRecord, authority, { configDir: home })).toThrow("trusted-route"); + expect(() => observationFromLiveResult(result, caseRecord, authority, { configDir: home })).toThrow("trusted execution receipt"); + }); + + test("fabricated trusted-route result cannot create live evidence", () => { + const home = tempHome(); const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases[0]!; + const fake: LiveScenarioRunResult = { scenarioId: caseRecord.id, suite: caseRecord.suite, startedAt: 1, completedAt: 2, passed: true, classification: "inconclusive", assertionResults: [], diagnostics: [], routeSubject: routeSubject(), executionAuthority: "trusted_route" }; + expect(() => observationFromLiveResult(fake, caseRecord, authority, { configDir: home })).toThrow("trusted execution receipt"); + }); + + test("trusted receipt is bound to the executed scenario and authority", async () => { + const home = tempHome(); const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); + const differentCase = authority.cases.find((c) => c.id === "tools-core.live.function-round-trip")!; + expect(() => observationFromLiveResult(result, differentCase, authority, { configDir: home })).toThrow("receipt mismatch"); + }); + + test("unapproved routes do not resolve destinations", async () => { + const scenario = loadLiveCaseAuthority().cases[0]!; let resolves = 0; + const result = await runLiveScenario(scenario, mockRoute({ labRunApproval: false }), { resolve: async () => { resolves += 1; return [{ address: "93.184.216.34", family: 4 }]; }, transport: transportForCase(scenario) }); + expect(resolves).toBe(0); expect(result.secondaryCode).toBe("route_precondition_unmet:lab_run_approval"); expect(result.routeSubject).toBeUndefined(); + }); + + test("executor diagnostics expose only bounded codes", async () => { + const scenario = loadLiveCaseAuthority().cases[0]!; const secret = "Bearer super-secret-value"; + const result = await runLiveScenario(scenario, mockRoute(), { resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: createHostIssuedLabRouteExecutor(async () => { throw new Error(secret); }) }); + expect(result.diagnostics).toEqual(["execution_error"]); expect(JSON.stringify(result)).not.toContain(secret); }); test("raw URLs and secrets never appear in persisted trusted live evidence", async () => { @@ -116,11 +145,26 @@ describe("CL-03 live probe harness", () => { test("freshness semantics for live suites", () => { const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); expect(suite.freshness.maxAgeMs).toBe(604800000); - const subject: RouteSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "route", providerId: "p", providerInstanceFingerprint: "a".repeat(64), clientModelId: "m", upstreamModelId: "m", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; - const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject, loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + const subject = routeSubject(); + const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject, routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); expect(result.canVerify).toBe(false); expect(result.missingRequiredScenarioIds.length).toBeGreaterThan(0); }); + test("live verification fails closed without route identity or validated claims", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); + const protocolSubject: ProtocolSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "protocol", opencodexCompatibilityVersion: "a".repeat(64), effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", behaviorFingerprint: "b".repeat(64) }; + const noSubject = evaluateAllApplicableRequiredPassV1(suite, [], "live", { routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + const wrongSubject = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: protocolSubject, routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + const noClaims = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: routeSubject(), loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + expect(noSubject.notes).toContain("route_subject_required"); expect(wrongSubject.notes).toContain("route_subject_required"); expect(noClaims.notes).toContain("route_claim_state_required"); + }); + + test("malformed requiredClaims in a live manifest is rejected", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); + const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: routeSubject(), routeSupportedClaims: [], loadScenarioManifest: () => ({ requirements: { inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: "tools" }, freshness: { maxAgeMs: 604800000 } }) }); + expect(result.canVerify).toBe(false); expect(result.notes.some((note) => note.startsWith("scenario_manifest_unavailable:"))).toBe(true); + }); + test("timeouts classify as blockers", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const scenario = loadLiveCaseAuthority().cases[0]!; const result = await runLiveScenario(scenario, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: "{}", error: "total_timeout" }] }) }); From 1b3ea0470d7d6881ab57687819a398c442b1beaa Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:15:03 +0200 Subject: [PATCH 25/47] test(lab): reject forged live executor authority --- tests/lab-live-sandbox.test.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index dd5f0ad55..7d54899fd 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -5,10 +5,13 @@ import { tmpdir } from "node:os"; import { assertDestinationAddressSet, assertHostSniMatch, assertLeaseScope, buildRouteSubjectV1, classifyTransportError, createCredentialLease, createLabDestination, createMockTransport, createSandboxResourceState, enforceSandboxLimits, - freezeRouteSubject, LabCredentialError, LabDestinationError, LabSandboxError, prepareLiveSandbox, rejectProxyEnvironment, TransportError, + freezeRouteSubject, LabCredentialError, LabDestinationError, LabSandboxError, prepareLiveSandbox, readInstallationSalt, + rejectProxyEnvironment, TransportError, } from "../src/lab"; -import { createTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; -import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext } from "../src/lab/live/types"; +import * as executionAuthority from "../src/lib/lab-live-execution-authority"; +import { isTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext, type TrustedLabRouteExecutor } from "../src/lab/live/types"; const HOMES: string[] = []; function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } @@ -44,12 +47,18 @@ describe("CL-03 live sandbox security seams", () => { test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); test("observable resource counters enforce child-process limit", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); }); - test("trusted executor cannot be constructed without the hard memory/process/time boundaries", () => { - const missingMemory = REQUIRED_LAB_SANDBOX_BOUNDARIES.filter((boundary) => boundary !== "memory_limit"); - expect(() => createTrustedLabRouteExecutor(async () => { throw new Error("never"); }, missingMemory)).toThrow("required sandbox enforcement"); - expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("memory_limit"); - expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("child_process_limit"); - expect(REQUIRED_LAB_SANDBOX_BOUNDARIES).toContain("connect_timeout"); + test("untrusted runtime code cannot self-attest a trusted executor", async () => { + expect("createTrustedLabRouteExecutor" in executionAuthority).toBe(false); + const fake = { execute: async () => { throw new Error("never"); }, enforcedBoundaries: [...REQUIRED_LAB_SANDBOX_BOUNDARIES] } as TrustedLabRouteExecutor; + expect(isTrustedLabRouteExecutor(fake)).toBe(false); + const hostIssued = createHostIssuedLabRouteExecutor(async () => { throw new Error("never"); }); + expect(isTrustedLabRouteExecutor(hostIssued)).toBe(true); + expect(hostIssued.enforcedBoundaries).toEqual(REQUIRED_LAB_SANDBOX_BOUNDARIES); + }); + + test("installation salt is stable across repeated reads", () => { + const home = tempHome(); const first = readInstallationSalt(home); const second = readInstallationSalt(home); + expect(first).toEqual(second); expect(first.byteLength).toBe(32); }); test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); From fdc2a69d0f9be7effaa21346bad27385745370c9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:17:32 +0200 Subject: [PATCH 26/47] fix(lab): close expected failure classification --- src/lab/conformance/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 872accc19..30cc0c963 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -38,7 +38,7 @@ export interface AssertionSpec { export interface ExpectedFailureSpec { controlKind: "conformance_negative_control" | "capability_absence_control"; - expectedClass: string; + expectedClass: FailureClassification; expectedCode: string; assertionIds: string[]; onMatch: "pass" | "unsupported"; From b6ef4edc539ed8283f33822c8d9c3523ce9899a2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:18:11 +0200 Subject: [PATCH 27/47] fix(lab): bound destination resolution by connect timeout --- src/lab/live/destination.ts | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts index 4a3f4a994..0be4a2c56 100644 --- a/src/lab/live/destination.ts +++ b/src/lab/live/destination.ts @@ -51,11 +51,26 @@ function addressSetKey(addresses: ReadonlyArray<{ address: string; family: 4 | 6 return canonicalAddresses(addresses).map((row) => `${row.family}:${row.address}`).join(","); } +async function withResolutionTimeout(timeoutMs: number, operation: () => Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + operation(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new LabDestinationError("destination resolution timed out", "connect_timeout")), timeoutMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + export interface CreateDestinationOptions { baseUrl: string; allowPrivateNetwork?: boolean; labRunApproval?: boolean; resolve?: DnsResolver; + connectTimeoutMs?: number; configDir?: string; } @@ -79,22 +94,31 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom throw new LabDestinationError("private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); } + const connectTimeoutMs = opts.connectTimeoutMs ?? 10_000; + if (!Number.isInteger(connectTimeoutMs) || connectTimeoutMs <= 0) { + throw new LabDestinationError("invalid connect timeout", "harness_failure"); + } + let addresses: Array<{ address: string; family: 4 | 6 }>; let privateNetwork = literalPrivate; if (opts.resolve) { - try { addresses = await opts.resolve(parsed.hostname); } - catch { throw new LabDestinationError("DNS resolution failed", "network_blocked"); } + try { addresses = await withResolutionTimeout(connectTimeoutMs, () => opts.resolve!(parsed.hostname)); } + catch (error) { + if (error instanceof LabDestinationError) throw error; + throw new LabDestinationError("DNS resolution failed", "network_blocked"); + } if (addresses.length === 0) throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); for (const row of addresses) privateNetwork = validateResolvedAddress(scheme, row, allowPrivate, approved) || privateNetwork; } else { try { - const resolved = await resolvePublicAddresses(parsed.toString(), { + const resolved = await withResolutionTimeout(connectTimeoutMs, () => resolvePublicAddresses(parsed.toString(), { context: "Lab provider destination", allowPrivateNetwork: allowPrivate && approved, - }); + })); addresses = resolved.addresses.map((row) => ({ address: row.address, family: row.family === 6 ? 6 as const : 4 as const })); privateNetwork = resolved.privateNetwork || privateNetwork; - } catch { + } catch (error) { + if (error instanceof LabDestinationError) throw error; throw new LabDestinationError("DNS destination policy failed", "network_blocked"); } } From 62196b0ede127f5d69326541d0290fb87ee39a9b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:20:00 +0200 Subject: [PATCH 28/47] test(lab): cover destination timeout and canonicalization --- tests/lab-live-sandbox.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index 7d54899fd..2b18e85d8 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -38,6 +38,17 @@ describe("CL-03 live sandbox security seams", () => { const lease = createCredentialLease({ destination, budget: 1 }); expect(() => assertLeaseScope(lease, other)).toThrow(LabCredentialError); }); + test("address-set equality uses the same canonical representation on both sides", async () => { + const home = tempHome(); const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "2001:db8::1", family: 6 }, { address: "93.184.216.34", family: 4 }, { address: "93.184.216.34", family: 4 }], configDir: home }); + expect(() => assertDestinationAddressSet(destination, [{ address: "93.184.216.34", family: 4 }, { address: "2001:db8::1", family: 6 }])).not.toThrow(); + }); + + test("destination resolution obeys the connect timeout", async () => { + const home = tempHome(); + const pending = createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, connectTimeoutMs: 5, resolve: async () => await new Promise>(() => {}), configDir: home }); + await expect(pending).rejects.toMatchObject({ code: "connect_timeout" }); + }); + test("host/SNI mismatch fails closed", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); expect(() => assertHostSniMatch(destination, "evil.example.com")).toThrow(LabDestinationError); }); test("metadata and link-local destinations are blocked", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://169.254.169.254/latest/meta-data", labRunApproval: true, configDir: home })).rejects.toThrow(LabDestinationError); }); test("private network requires permission and lab approval", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", configDir: home })).rejects.toThrow(LabDestinationError); const ok = await createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", allowPrivateNetwork: true, labRunApproval: true, resolve: async () => [{ address: "127.0.0.1", family: 4 }], configDir: home }); expect(ok.privateNetwork).toBe(true); }); From ec89264babb032a8ca43b4b782de1ecbdf83f985 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:21:01 +0200 Subject: [PATCH 29/47] fix(lab): keep shared pinned connect timeout opt-in --- src/lib/pinned-http.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/pinned-http.ts b/src/lib/pinned-http.ts index 09b361fda..03c9f8f69 100644 --- a/src/lib/pinned-http.ts +++ b/src/lib/pinned-http.ts @@ -6,7 +6,7 @@ export type PinnedAddress = { address: string; family: number }; export interface PinnedHttpRequestOptions { headers?: HeadersInit; maxBytes?: number; - /** Deadline for establishing the TCP connection and, for HTTPS, completing TLS. */ + /** Optional deadline for establishing the TCP connection and, for HTTPS, completing TLS. */ connectTimeoutMs?: number; idleTimeoutMs?: number; rejectUnauthorized?: boolean; @@ -29,7 +29,7 @@ function pinnedHttpRequest( throw new Error(`${options?.context ?? "request"} must use HTTP or HTTPS, got ${parsed.protocol}`); } const context = options?.context ?? "request"; - const connectTimeoutMs = options?.connectTimeoutMs ?? 10_000; + const connectTimeoutMs = options?.connectTimeoutMs; const idleTimeoutMs = options?.idleTimeoutMs ?? 60_000; const maxBytes = options?.maxBytes; const headers = new Headers(options?.headers); @@ -156,7 +156,7 @@ function pinnedHttpRequest( const onAbort = () => fail(signal?.reason instanceof Error ? signal.reason : new Error("aborted")); signal?.addEventListener("abort", onAbort, { once: true }); req.on("socket", (socket) => { - if (!socket.connecting) return; + if (!socket.connecting || connectTimeoutMs === undefined) return; const connectedEvent = parsed.protocol === "https:" ? "secureConnect" : "connect"; connectTimer = setTimeout(() => fail(new Error(`${context} connect timed out`)), connectTimeoutMs); socket.once(connectedEvent, clearConnectTimer); From 562555c8f342657fe0eb8019fe6427a8ebe0b145 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:21:32 +0200 Subject: [PATCH 30/47] docs(structure): record CL-03 trust boundaries --- structure/09_compatibility-lab.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 structure/09_compatibility-lab.md diff --git a/structure/09_compatibility-lab.md b/structure/09_compatibility-lab.md new file mode 100644 index 000000000..5535f830a --- /dev/null +++ b/structure/09_compatibility-lab.md @@ -0,0 +1,21 @@ +# Compatibility Lab SOT + +## CL-03 live-route execution boundary + +CL-03 live-route evidence is generated only for an exact `RouteSubjectV1` and remains separate from protocol-conformance and task-effectiveness evidence. + +The live runner fails closed before destination resolution unless the selected scenario is applicable and every route precondition, including explicit `lab_run_approval`, is satisfied. Destination resolution is bounded by the CL-03 connect timeout, policy-checks every resolved address, freezes the approved address set, and fingerprints only the immutable destination snapshot. Raw URLs and resolved addresses are not persisted as Lab evidence. + +Evidence-eligible route execution uses a host-issued `TrustedLabRouteExecutor`. The public Lab authority surface only recognizes host-issued capabilities; it does not expose a constructor that accepts caller-asserted sandbox boundary names. Test transports remain useful for normalization/classification tests but are never evidence-eligible. + +Successful or blocked trusted executions receive a module-private receipt bound to the canonical live authority, scenario ID, suite ID, scenario/suite manifest digests, and exact route subject ID. `observationFromLiveResult` verifies that receipt before creating directories or writing artifacts, so a structural `LiveScenarioRunResult` or mismatched case/authority cannot fabricate live evidence. + +The trusted credential sender keeps secret injection outside Lab code and uses the existing pinned HTTP primitive. CL-03 explicitly supplies its connect timeout; other pinned-HTTP callers retain their prior timeout behavior. Only response metadata required by live assertions currently crosses back into Lab (`content-type`); cookies, account/organization metadata, credential-adjacent headers, and rate-limit headers are not exposed. + +Live projection preserves the frozen `RouteSubjectV1` schema. Claim-gated scenario applicability is derived from current validated, usable `claim_snapshot` state for the exact subject rather than from caller-provided claim arrays or by extending the V1 subject preimage. A missing/wrong-kind route subject or unavailable claim state fails verification closed. + +The two machine-readable Live V1 authority copies are required to be byte-identical. Runtime loading fails closed on byte drift before parsing. Scenario limits use `perArtifactBytes` as the single per-artifact execution-limit key; the artifact policy retains its independent per-artifact policy ceiling. + +## Scope guard + +CL-03 does not expose a management CLI/API or UI. Those surfaces remain CL-04+ work. Production request routing must not synchronously trigger Compatibility Lab probing or rebuild Lab evidence. From 7eb76e6b1c6dfa02f57e43f52b272d8a097083b4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:22:56 +0200 Subject: [PATCH 31/47] fix(lab): classify destination connect timeout as blocker --- src/lab/live/destination.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts index 0be4a2c56..56d9bc444 100644 --- a/src/lab/live/destination.ts +++ b/src/lab/live/destination.ts @@ -2,6 +2,7 @@ import { isIP } from "node:net"; import { assessUrlDestination, resolvePublicAddresses } from "../../lib/destination-policy"; import { localFingerprint } from "../digest"; import { readInstallationSalt } from "../subject/installation-salt"; +import { TransportError } from "./transport"; import type { DnsResolver, LabDestinationV1 } from "./types"; export class LabDestinationError extends Error { @@ -57,7 +58,7 @@ async function withResolutionTimeout(timeoutMs: number, operation: () => Prom return await Promise.race([ operation(), new Promise((_, reject) => { - timer = setTimeout(() => reject(new LabDestinationError("destination resolution timed out", "connect_timeout")), timeoutMs); + timer = setTimeout(() => reject(new TransportError("connect_timeout", "destination resolution timed out")), timeoutMs); }), ]); } finally { @@ -104,7 +105,7 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom if (opts.resolve) { try { addresses = await withResolutionTimeout(connectTimeoutMs, () => opts.resolve!(parsed.hostname)); } catch (error) { - if (error instanceof LabDestinationError) throw error; + if (error instanceof TransportError || error instanceof LabDestinationError) throw error; throw new LabDestinationError("DNS resolution failed", "network_blocked"); } if (addresses.length === 0) throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); @@ -118,7 +119,7 @@ export async function createLabDestination(opts: CreateDestinationOptions): Prom addresses = resolved.addresses.map((row) => ({ address: row.address, family: row.family === 6 ? 6 as const : 4 as const })); privateNetwork = resolved.privateNetwork || privateNetwork; } catch (error) { - if (error instanceof LabDestinationError) throw error; + if (error instanceof TransportError || error instanceof LabDestinationError) throw error; throw new LabDestinationError("DNS destination policy failed", "network_blocked"); } } From ae590ec504d847ba954da9c48431bfca44139255 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:26:00 +0200 Subject: [PATCH 32/47] fix(lab): publish installation salt after durable staging write --- src/lab/subject/installation-salt.ts | 63 +++++++++++++++++++++------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts index df219889d..dfc804537 100644 --- a/src/lab/subject/installation-salt.ts +++ b/src/lab/subject/installation-salt.ts @@ -1,27 +1,60 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { closeSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; +import { dirname } from "node:path"; import { labInstallationSaltPath, labRoot } from "../paths"; const SALT_BYTES = 32; -/** Read or create the per-installation salt used for local fingerprinting. */ +function readSaltFile(path: string): Uint8Array { + const bytes = readFileSync(path); + if (bytes.byteLength !== SALT_BYTES) { + throw new Error("harness_failure: invalid installation salt length"); + } + return new Uint8Array(bytes); +} + +function removeStagingFile(path: string): void { + try { unlinkSync(path); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +/** Read or atomically publish the per-installation salt used for local fingerprinting. */ export function readInstallationSalt(configDir?: string): Uint8Array { const path = labInstallationSaltPath(configDir); - const readExisting = (): Uint8Array => { - const bytes = readFileSync(path); - if (bytes.byteLength !== SALT_BYTES) { - throw new Error("harness_failure: invalid installation salt length"); - } - return new Uint8Array(bytes); - }; + const root = labRoot(configDir); + mkdirSync(root, { recursive: true, mode: 0o700 }); + + try { return readSaltFile(path); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } - mkdirSync(labRoot(configDir), { recursive: true, mode: 0o700 }); + // The final pathname must never become visible before all 32 bytes are durable in the + // staging inode. A hard-link publish is atomic and fails with EEXIST when another process won. + const stagingPath = `${path}.tmp-${process.pid}-${randomBytes(8).toString("hex")}`; + if (dirname(stagingPath) !== dirname(path)) throw new Error("harness_failure: installation salt staging path escaped directory"); const salt = randomBytes(SALT_BYTES); + let fd: number | undefined; try { - writeFileSync(path, salt, { mode: 0o600, flag: "wx" }); - return new Uint8Array(salt); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - return readExisting(); + fd = openSync(stagingPath, "wx", 0o600); + writeFileSync(fd, salt); + fsyncSync(fd); + closeSync(fd); + fd = undefined; + + try { + linkSync(stagingPath, path); + return new Uint8Array(salt); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return readSaltFile(path); + } + } finally { + if (fd !== undefined) { + try { closeSync(fd); } catch { /* best-effort close before cleanup */ } + } + removeStagingFile(stagingPath); } } From 161048c05b0fd08ae250316ee396d6883007dd73 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:26:36 +0200 Subject: [PATCH 33/47] test(lab): cover multiprocess salt publication --- tests/lab-live-sandbox.test.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index 2b18e85d8..63831fc5b 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync } from "node:fs"; +import { spawn } from "node:child_process"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { @@ -26,6 +27,21 @@ function baseRoute(overrides: Partial = {}): LabRouteContext { } const limits = { totalTimeoutMs: 120000, connectTimeoutMs: 10000, firstByteTimeoutMs: 30000, inactivityTimeoutMs: 30000, maxRequests: 16, maxInputBytes: 8388608, maxOutputBytes: 16777216, maxOutputTokens: 32768, maxToolCalls: 32, maxMemoryBytes: 536870912, maxChildProcesses: 0, maxArtifacts: 16, perArtifactBytes: 262144, aggregateArtifactBytes: 1048576 }; +function runSaltInitializer(script: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; let stderr = ""; + child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { stdout += chunk; }); + child.stderr.on("data", (chunk: string) => { stderr += chunk; }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(stdout.trim()); + else reject(new Error(`salt initializer exited ${code}: ${stderr.trim()}`)); + }); + }); +} + describe("CL-03 live sandbox security seams", () => { test("rejects proxy environment variables", () => { expect(() => rejectProxyEnvironment({ HTTP_PROXY: "http://evil" })).toThrow(LabSandboxError); expect(() => rejectProxyEnvironment({ no_proxy: "localhost" })).toThrow(LabSandboxError); }); test("prepareLiveSandbox allows only TZ=UTC and NO_COLOR=1", () => { expect(prepareLiveSandbox({})).toEqual({ TZ: "UTC", NO_COLOR: "1" }); }); @@ -72,6 +88,15 @@ describe("CL-03 live sandbox security seams", () => { expect(first).toEqual(second); expect(first.byteLength).toBe(32); }); + test("concurrent first-run salt initialization converges on one completed salt", async () => { + const home = tempHome(); + const saltModule = new URL("../src/lab/subject/installation-salt.ts", import.meta.url).href; + const script = `import { readInstallationSalt } from ${JSON.stringify(saltModule)};\nconst salt = readInstallationSalt(${JSON.stringify(home)});\nprocess.stdout.write(Buffer.from(salt).toString("hex"));`; + const [first, second] = await Promise.all([runSaltInitializer(script), runSaltInitializer(script)]); + expect(first).toHaveLength(64); expect(second).toBe(first); + expect(Buffer.from(readInstallationSalt(home)).toString("hex")).toBe(first); + }); + test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); test("route subject stability across credential rotation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); createCredentialLease({ destination, budget: 1 }).consume(); const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); expect(subjectA).toEqual(subjectB); }); }); From 8308c9ff00259dd18a6f4eec538be4ce2be1e904 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:54:26 +0200 Subject: [PATCH 34/47] docs(lab): correct CL-03 focused test totals --- .../001_pr_stack_status.md | 175 +----------------- 1 file changed, 3 insertions(+), 172 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index b88a4f4b0..1af4d4bea 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -1,172 +1,3 @@ -# Compatibility Lab PR stack status - -Updated throughout the programme. Every phase records its branch, exact -starting/base revision, accepted contract/implementation head, PR, verification, -independent review, blockers, and whether a later phase is authorized. - -## Programme facts - -- Repository: `lidge-jun/opencodex` -- Integration target: `dev` -- CL-00 starting `upstream/dev`: - `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` -- Package/runtime at start: OpenCodex `2.10.2`, Bun `1.3.14` -- CL-00 branch: `feat/cl-00-compatibility-contracts` -- CL-00 scope: documentation/contracts/incident corpus only -- PR target: `lidge-jun/opencodex:dev` - -## Stack - -| Phase | Branch | Starting/base SHA | Accepted head | PR | State | -|---|---|---|---|---|---| -| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | NOT RECORDED | [#1333](https://github.com/lidge-jun/opencodex/pull/1333) | MERGED TO `dev` at `025c37916225dd685d9217e5b40190600f06d278`; POST-MERGE HARDENING [#1343](https://github.com/lidge-jun/opencodex/pull/1343) MERGED at `eee2dab4d1bbacefce56057adad51d734f346702`; FINAL CLOSURE GATE [#1348](https://github.com/lidge-jun/opencodex/pull/1348) | -| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | `003f7402f49bfe8dd710a7beba52f717051bfadf` | [#1352](https://github.com/lidge-jun/opencodex/pull/1352) | DRAFT PR OPEN (implementation; not accepted) | - -The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its -moving base-ref name is not a substitute for that historical SHA. - -CL-02 starts from the exact CL-01 merge commit on `dev` -(`4bb249b756abd468c675d2d92fffe4da95ad3e2a` / upstream #1320). It does **not** -base on the pre-merge CL-01 feature branch tip. - -## CL-00 acceptance log - -- Live-tree audit covered provider registry/derivation, Routing Profiles, - routing traces, request history/analytics, doctor/connectivity validation, - protocol regression tests, and relevant incident/devlog records. -- CL-00 remains contract-only. No Compatibility Lab runtime, live runner, - profile/router implementation, or CL-02 work was added. -- Contract documents: - - `000_master_plan.md` - - `010_architecture_and_evidence_contract.md` - - `020_scenario_contract_and_catalogue.md` - - `021_protocol_v1_manifest_authority.md` - - `022_protocol_v1_cases.json` - - `030_incident_corpus.md` - - `040_security_and_privacy.md` - - `050_cl00_acceptance_review.md` -- Original baseline verification: - - `bun x tsc --noEmit`: passed. - - `bun run privacy:scan`: passed. - - `tests/repo-hygiene.test.ts`: 11 passed, 0 failed. - - focused protocol/compatibility: 395 passed, 0 failed across 24 files. - - continuation semantics: 2 passed, 95 filtered, 0 failed. - - isolated cache invalidation: 6 passed, 0 failed. - - isolated native residue: 63 passed, 2 platform skips, 0 failed. - - full `bun run test` was not green on the Windows/Bun 1.3.14 host for the - previously documented cache/account/Bun panic failures; a broader - `responses-state` run also had four Windows `EPERM` symlink failures. - -Independent CL-00 acceptance review is frozen at -`c014464237fd3c95bda08bc18bfab8ba8f532308`. Merged to `dev` via #1286. - -## CL-01 contract-correction log (2026-08-09) - -- **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) -- **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` -- **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) -- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `22d608c82d82e2746c0cef9cd761db19a8e465ee` -- **Merged to `dev`:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` via upstream [#1320](https://github.com/lidge-jun/opencodex/pull/1320). - -### Corrections applied - -1. Rebased onto merged CL-00 / #1286 (`243c3f490`). -2. Synced `022_protocol_v1_cases.json` runtime copy with final CL-00 authority. -3. Removed Chat → Responses `input[]` observation projection. -4. Chat tool-result selectors: `/upstream/requests/1/json/messages/1/tool_call_id` for function-round-trip and apply-patch-turn. -5. SSE `[DONE]` normalization keyed by source protocol (`openai-chat` only). -6. Mandatory synthetic fixture marker/provenance in expanded manifests; fail-closed validation. -7. Four deterministic MCP action tokens in `mcp-stub.ts`. -8. Recomputed scenario manifest digests (provenance participates in JCS expansion). -9. Narrow image tool-result wire normalization for `tools-core.protocol.result-content` (indices only). -10. `openai-chat.ts`: `toolResultTextForWire` omits `[image]` marker when images are flushed to user carrier. - -### Verification (correction) - -- `bun x tsc --noEmit`: passed -- `bun test tests/lab-conformance-harness.test.ts`: 14/14 passed -- `git diff --check`: passed -- Independent review: `051_cl01_acceptance_review.md` — ACCEPTED (revalidation) - -### Blockers - -- None for CL-01 correction. -- Full-suite green remains unavailable on this host for documented Windows/Bun reasons. - -## CL-02 implementation log - -- **Branch:** `feat/cl-02-evidence-ledger` -- **Starting/base SHA:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` (CL-01 merge via #1320) -- **Merged source head:** `1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1` -- **Merged to `dev`:** `025c37916225dd685d9217e5b40190600f06d278` via upstream [#1333](https://github.com/lidge-jun/opencodex/pull/1333). -- **Accepted head:** not recorded. #1333 was merged before an independent-acceptance state was observed in this programme log. -- **Scope:** append-only JSONL evidence ledger with an explicit sensitive-purge - exception: when the `ledger` purge action is requested, targeted evidence is - physically removed by atomic ledger rewrite and a `purge_tombstone` remains as - the auditable record; SQLite is rebuilt from the rewritten ledger and retained - content-addressed artifacts. The phase also includes the content-addressed - artifact store, disposable/rebuildable SQLite projection, ClaimSourceManifestV1, - invalidation semantics, and the CL-01 → observation persistence seam. -- **Explicitly out of scope:** CL-03 live probes, CL-04 CLI/API, CL-05 UI, - CL-06 profile fields, Fabric, shadow workflows. - -### Boundary note (verdict algorithm) - -CL-02 implements frozen `all-applicable-required-pass-v1` evaluation with -subject-aware applicability (required scenarios whose manifest requirements -match the exact protocol subject, excluding live-reserved cases in fixture -mode). Positive `VERIFIED` requires a non-empty applicable required set and a -current pass for every member. Descriptor/handle-bound artifact I/O, fail-closed -sensitive purge with shared-artifact retention, recursive event admission -ceilings, and unusable-evidence exclusion from projection are implemented. -Claims cannot produce `PROBED`/`VERIFIED`. - -### CL-02 validation and post-merge hardening status (2026-08-09) - -- **Prior accepted review-fix head:** `cf626d14c823413fbcd6ac2625d1da16bbac714e` -- **Final #1333 source head:** `1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1` -- **#1333 merge commit:** `025c37916225dd685d9217e5b40190600f06d278` -- A final CodeRabbit review batch arrived immediately before the #1333 merge and - identified additional hardening work in ledger replay, artifact publication, - sensitive purge, event privacy admission, contract-artifact error - classification, conformance execution timestamps, and regression coverage. -- **Post-merge hardening branch:** `fix/cl-02-post-merge-hardening` -- **Post-merge hardening PR:** [#1343](https://github.com/lidge-jun/opencodex/pull/1343), based exactly on #1333 merge commit `025c37916225dd685d9217e5b40190600f06d278`. -- **Final #1343 source head:** `953e75f498056edabb9c4f2b33945f6a3d081780`. -- **#1343 merged to `dev`:** `eee2dab4d1bbacefce56057adad51d734f346702` on 2026-08-09. -- #1343 preserved frozen CL-00 semantics and did not add CL-03 work. -- React Doctor for final #1343 head passed. Cross-platform CI run `31305383591` - remained in progress when the closure follow-up was opened; its completed jobs - included green gates/typecheck/privacy and green test shards 1/4, 2/4, and 4/4. -- One final CodeRabbit test-quality finding remained on the merged PR: the - targetless purge validation regression asserted only `LabValidationError` - rather than the exact `empty_purge_targets` code. -- **Closure branch:** `fix/cl-02-closure-final`, based on then-current `dev` - `f197529c7d8c6adbaf3f859547414698d349340d`, which contains #1343. -- **Closure PR:** [#1348](https://github.com/lidge-jun/opencodex/pull/1348). -- #1348 strengthens the purge regression to assert the exact error code and - records the closure gate in `053_cl02_closure_gate.md`. -- **CL-03:** not started. Authorization requires #1348 green CI, zero unresolved - valid CodeRabbit findings, and merge to `dev`. - -## Authorization - -- CL-00: **ACCEPTED** (merged #1286). -- CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. -- CL-02: **MERGED** via #1333 at `025c37916225dd685d9217e5b40190600f06d278`; post-merge hardening #1343 is also **MERGED** at `eee2dab4d1bbacefce56057adad51d734f346702`; final closure is tracked in #1348. -- CL-03: **DRAFT PR OPEN** ([#1352](https://github.com/lidge-jun/opencodex/pull/1352)) on - `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`; - implementation head `003f7402f49bfe8dd710a7beba52f717051bfadf`. Not accepted. -- CL-04: **NOT STARTED** (blocked until CL-03 independent acceptance and review reconciliation). - -## CL-03 implementation log (2026-08-09) - -- **Branch:** `feat/cl-03-live-route-probes` -- **Starting/base SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (#1348 merge on `dev`) -- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` -- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) - **Scope:** bounded live-route probes for `live_route_compatibility`; live manifest authority; `RouteSubjectV1` builder; `LabDestinationV1` / credential lease sandbox; inert tool/MCP stubs; live runner/executor; `observe/from-live` persistence; @@ -180,8 +11,8 @@ Claims cannot produce `PROBED`/`VERIFIED`. - `bun test tests/lab-conformance-harness.test.ts`: 17/17 passed - `bun test tests/lab-evidence-ledger.test.ts`: 37/41 passed (4 Windows SQLite `EPERM` flakes in `wipeSqlite`; `rebuild.ts` unchanged vs `upstream/dev` — pre-existing) -- `bun test tests/lab-live-probe.test.ts`: 12/12 passed -- `bun test tests/lab-live-sandbox.test.ts`: 12/12 passed +- `bun test tests/lab-live-probe.test.ts`: 19/19 passed +- `bun test tests/lab-live-sandbox.test.ts`: 17/17 passed - `bun run privacy:scan`: passed - Cross-platform CI on #1352: pending at open time @@ -189,4 +20,4 @@ Claims cannot produce `PROBED`/`VERIFIED`. - Independent acceptance review not performed - Draft PR review findings not yet reconciled -- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) +- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) \ No newline at end of file From c6117a59dc8be4435699aeda86b5db86a3df9f1b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:54:38 +0200 Subject: [PATCH 35/47] docs(lab): correct CL-03 implementation test totals --- .../054_cl03_implementation_record.md | 40 ++----------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md index cffef1568..df05391b6 100644 --- a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md +++ b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md @@ -1,37 +1,3 @@ -# CL-03 implementation record - -## Branch and revisions - -- **Branch:** `feat/cl-03-live-route-probes` -- **Starting SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) -- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` -- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) - -## Scope delivered - -1. Live manifest authority (`023_live_v1_manifest_authority.md`, `024_live_v1_cases.json`, runtime copy) -2. Route subject builder (`src/lab/subject/`) -3. Live sandbox (`src/lab/live/` — destination, credential lease, transport, executor, runner) -4. Persistence seam (`src/lab/observe/from-live.ts`) -5. Projection applicability (`routeSubjectApplicableToRequirements` in `verification.ts`) -6. Tests (`tests/lab-live-sandbox.test.ts`, `tests/lab-live-probe.test.ts`) - -## Security design summary - -- **Network:** immutable in-memory `LabDestinationV1` snapshot; DNS/policy before credentials; - connect only to approved address set; redirects rejected; metadata/link-local forbidden; - private/loopback requires route `allowPrivateNetwork` plus explicit `labRunApproval`; - inherited proxy env vars rejected. -- **Credentials:** opaque `LabCredentialLeaseV1` bound to destination + transport + budget; - no secret bytes in Lab code; auth failure → `authentication_blocked` / `BLOCKED`. -- **Tools/MCP:** inert Lab-authored tools; loopback MCP stub only; no user `mcpServers`. -- **Process:** sandbox env `TZ=UTC`, `NO_COLOR=1` only; no child processes; resource ceilings enforced. - -## Frozen live scenarios (10) - -- `responses-core.live.basic-turn` -- `chat-core.live.basic-turn` -- `anthropic-core.live.basic-turn` - `tools-core.live.function-round-trip` - `tools-core.live.custom-freeform-round-trip` - `codex-core.live.tool-turn` @@ -47,8 +13,8 @@ | `bun x tsc --noEmit` | pass | | `tests/lab-conformance-harness.test.ts` | 17/17 | | `tests/lab-evidence-ledger.test.ts` | 37/41 (4 Windows SQLite EPERM flakes; pre-existing) | -| `tests/lab-live-probe.test.ts` | 12/12 | -| `tests/lab-live-sandbox.test.ts` | 12/12 | +| `tests/lab-live-probe.test.ts` | 19/19 | +| `tests/lab-live-sandbox.test.ts` | 17/17 | | `bun run privacy:scan` | pass | ## Explicitly not started @@ -60,4 +26,4 @@ ## Acceptance status Implementation only — **not accepted**. CL-04 remains blocked until independent CL-03 -acceptance and review reconciliation. +acceptance and review reconciliation. \ No newline at end of file From 07b40fe803bedb20f695682c64792701f902654f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:55:39 +0200 Subject: [PATCH 36/47] docs(lab): restore CL-03 implementation record and test totals --- .../054_cl03_implementation_record.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md index df05391b6..856abbbce 100644 --- a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md +++ b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md @@ -1,3 +1,37 @@ +# CL-03 implementation record + +## Branch and revisions + +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) + +## Scope delivered + +1. Live manifest authority (`023_live_v1_manifest_authority.md`, `024_live_v1_cases.json`, runtime copy) +2. Route subject builder (`src/lab/subject/`) +3. Live sandbox (`src/lab/live/` — destination, credential lease, transport, executor, runner) +4. Persistence seam (`src/lab/observe/from-live.ts`) +5. Projection applicability (`routeSubjectApplicableToRequirements` in `verification.ts`) +6. Tests (`tests/lab-live-sandbox.test.ts`, `tests/lab-live-probe.test.ts`) + +## Security design summary + +- **Network:** immutable in-memory `LabDestinationV1` snapshot; DNS/policy before credentials; + connect only to approved address set; redirects rejected; metadata/link-local forbidden; + private/loopback requires route `allowPrivateNetwork` plus explicit `labRunApproval`; + inherited proxy env vars rejected. +- **Credentials:** opaque `LabCredentialLeaseV1` bound to destination + transport + budget; + no secret bytes in Lab code; auth failure → `authentication_blocked` / `BLOCKED`. +- **Tools/MCP:** inert Lab-authored tools; loopback MCP stub only; no user `mcpServers`. +- **Process:** sandbox env `TZ=UTC`, `NO_COLOR=1` only; no child processes; resource ceilings enforced. + +## Frozen live scenarios (10) + +- `responses-core.live.basic-turn` +- `chat-core.live.basic-turn` +- `anthropic-core.live.basic-turn` - `tools-core.live.function-round-trip` - `tools-core.live.custom-freeform-round-trip` - `codex-core.live.tool-turn` @@ -26,4 +60,4 @@ ## Acceptance status Implementation only — **not accepted**. CL-04 remains blocked until independent CL-03 -acceptance and review reconciliation. \ No newline at end of file +acceptance and review reconciliation. From a80f79a418106e13fa78300e05f7127b1f3ee19f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:56:21 +0200 Subject: [PATCH 37/47] docs(lab): restore PR stack status and test totals --- .../001_pr_stack_status.md | 171 +++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 1af4d4bea..1ce0942d0 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -1,3 +1,172 @@ +# Compatibility Lab PR stack status + +Updated throughout the programme. Every phase records its branch, exact +starting/base revision, accepted contract/implementation head, PR, verification, +independent review, blockers, and whether a later phase is authorized. + +## Programme facts + +- Repository: `lidge-jun/opencodex` +- Integration target: `dev` +- CL-00 starting `upstream/dev`: + `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` +- Package/runtime at start: OpenCodex `2.10.2`, Bun `1.3.14` +- CL-00 branch: `feat/cl-00-compatibility-contracts` +- CL-00 scope: documentation/contracts/incident corpus only +- PR target: `lidge-jun/opencodex:dev` + +## Stack + +| Phase | Branch | Starting/base SHA | Accepted head | PR | State | +|---|---|---|---|---|---| +| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | NOT RECORDED | [#1333](https://github.com/lidge-jun/opencodex/pull/1333) | MERGED TO `dev` at `025c37916225dd685d9217e5b40190600f06d278`; POST-MERGE HARDENING [#1343](https://github.com/lidge-jun/opencodex/pull/1343) MERGED at `eee2dab4d1bbacefce56057adad51d734f346702`; FINAL CLOSURE GATE [#1348](https://github.com/lidge-jun/opencodex/pull/1348) | +| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | `003f7402f49bfe8dd710a7beba52f717051bfadf` | [#1352](https://github.com/lidge-jun/opencodex/pull/1352) | DRAFT PR OPEN (implementation; not accepted) | + +The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its +moving base-ref name is not a substitute for that historical SHA. + +CL-02 starts from the exact CL-01 merge commit on `dev` +(`4bb249b756abd468c675d2d92fffe4da95ad3e2a` / upstream #1320). It does **not** +base on the pre-merge CL-01 feature branch tip. + +## CL-00 acceptance log + +- Live-tree audit covered provider registry/derivation, Routing Profiles, + routing traces, request history/analytics, doctor/connectivity validation, + protocol regression tests, and relevant incident/devlog records. +- CL-00 remains contract-only. No Compatibility Lab runtime, live runner, + profile/router implementation, or CL-02 work was added. +- Contract documents: + - `000_master_plan.md` + - `010_architecture_and_evidence_contract.md` + - `020_scenario_contract_and_catalogue.md` + - `021_protocol_v1_manifest_authority.md` + - `022_protocol_v1_cases.json` + - `030_incident_corpus.md` + - `040_security_and_privacy.md` + - `050_cl00_acceptance_review.md` +- Original baseline verification: + - `bun x tsc --noEmit`: passed. + - `bun run privacy:scan`: passed. + - `tests/repo-hygiene.test.ts`: 11 passed, 0 failed. + - focused protocol/compatibility: 395 passed, 0 failed across 24 files. + - continuation semantics: 2 passed, 95 filtered, 0 failed. + - isolated cache invalidation: 6 passed, 0 failed. + - isolated native residue: 63 passed, 2 platform skips, 0 failed. + - full `bun run test` was not green on the Windows/Bun 1.3.14 host for the + previously documented cache/account/Bun panic failures; a broader + `responses-state` run also had four Windows `EPERM` symlink failures. + +Independent CL-00 acceptance review is frozen at +`c014464237fd3c95bda08bc18bfab8ba8f532308`. Merged to `dev` via #1286. + +## CL-01 contract-correction log (2026-08-09) + +- **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) +- **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` +- **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) +- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `22d608c82d82e2746c0cef9cd761db19a8e465ee` +- **Merged to `dev`:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` via upstream [#1320](https://github.com/lidge-jun/opencodex/pull/1320). + +### Corrections applied + +1. Rebased onto merged CL-00 / #1286 (`243c3f490`). +2. Synced `022_protocol_v1_cases.json` runtime copy with final CL-00 authority. +3. Removed Chat → Responses `input[]` observation projection. +4. Chat tool-result selectors: `/upstream/requests/1/json/messages/1/tool_call_id` for function-round-trip and apply-patch-turn. +5. SSE `[DONE]` normalization keyed by source protocol (`openai-chat` only). +6. Mandatory synthetic fixture marker/provenance in expanded manifests; fail-closed validation. +7. Four deterministic MCP action tokens in `mcp-stub.ts`. +8. Recomputed scenario manifest digests (provenance participates in JCS expansion). +9. Narrow image tool-result wire normalization for `tools-core.protocol.result-content` (indices only). +10. `openai-chat.ts`: `toolResultTextForWire` omits `[image]` marker when images are flushed to user carrier. + +### Verification (correction) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: 14/14 passed +- `git diff --check`: passed +- Independent review: `051_cl01_acceptance_review.md` — ACCEPTED (revalidation) + +### Blockers + +- None for CL-01 correction. +- Full-suite green remains unavailable on this host for documented Windows/Bun reasons. + +## CL-02 implementation log + +- **Branch:** `feat/cl-02-evidence-ledger` +- **Starting/base SHA:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` (CL-01 merge via #1320) +- **Merged source head:** `1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1` +- **Merged to `dev`:** `025c37916225dd685d9217e5b40190600f06d278` via upstream [#1333](https://github.com/lidge-jun/opencodex/pull/1333). +- **Accepted head:** not recorded. #1333 was merged before an independent-acceptance state was observed in this programme log. +- **Scope:** append-only JSONL evidence ledger with an explicit sensitive-purge + exception: when the `ledger` purge action is requested, targeted evidence is + physically removed by atomic ledger rewrite and a `purge_tombstone` remains as + the auditable record; SQLite is rebuilt from the rewritten ledger and retained + content-addressed artifacts. The phase also includes the content-addressed + artifact store, disposable/rebuildable SQLite projection, ClaimSourceManifestV1, + invalidation semantics, and the CL-01 → observation persistence seam. +- **Explicitly out of scope:** CL-03 live probes, CL-04 CLI/API, CL-05 UI, + CL-06 profile fields, Fabric, shadow workflows. + +### Boundary note (verdict algorithm) + +CL-02 implements frozen `all-applicable-required-pass-v1` evaluation with +subject-aware applicability (required scenarios whose manifest requirements +match the exact protocol subject, excluding live-reserved cases in fixture +mode). Positive `VERIFIED` requires a non-empty applicable required set and a +current pass for every member. Descriptor/handle-bound artifact I/O, fail-closed +sensitive purge with shared-artifact retention, recursive event admission +ceilings, and unusable-evidence exclusion from projection are implemented. +Claims cannot produce `PROBED`/`VERIFIED`. + +### CL-02 validation and post-merge hardening status (2026-08-09) + +- **Prior accepted review-fix head:** `cf626d14c823413fbcd6ac2625d1da16bbac714e` +- **Final #1333 source head:** `1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1` +- **#1333 merge commit:** `025c37916225dd685d9217e5b40190600f06d278` +- A final CodeRabbit review batch arrived immediately before the #1333 merge and + identified additional hardening work in ledger replay, artifact publication, + sensitive purge, event privacy admission, contract-artifact error + classification, conformance execution timestamps, and regression coverage. +- **Post-merge hardening branch:** `fix/cl-02-post-merge-hardening` +- **Post-merge hardening PR:** [#1343](https://github.com/lidge-jun/opencodex/pull/1343), based exactly on #1333 merge commit `025c37916225dd685d9217e5b40190600f06d278`. +- **Final #1343 source head:** `953e75f498056edabb9c4f2b33945f6a3d081780`. +- **#1343 merged to `dev`:** `eee2dab4d1bbacefce56057adad51d734f346702` on 2026-08-09. +- #1343 preserved frozen CL-00 semantics and did not add CL-03 work. +- React Doctor for final #1343 head passed. Cross-platform CI run `31305383591` + remained in progress when the closure follow-up was opened; its completed jobs + included green gates/typecheck/privacy and green test shards 1/4, 2/4, and 4/4. +- One final CodeRabbit test-quality finding remained on the merged PR: the + targetless purge validation regression asserted only `LabValidationError` + rather than the exact `empty_purge_targets` code. +- **Closure branch:** `fix/cl-02-closure-final`, based on then-current `dev` + `f197529c7d8c6adbaf3f859547414698d349340d`, which contains #1343. +- **Closure PR:** [#1348](https://github.com/lidge-jun/opencodex/pull/1348). +- #1348 strengthens the purge regression to assert the exact error code and + records the closure gate in `053_cl02_closure_gate.md`. +- **CL-03:** not started. Authorization requires #1348 green CI, zero unresolved + valid CodeRabbit findings, and merge to `dev`. + +## Authorization + +- CL-00: **ACCEPTED** (merged #1286). +- CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. +- CL-02: **MERGED** via #1333 at `025c37916225dd685d9217e5b40190600f06d278`; post-merge hardening #1343 is also **MERGED** at `eee2dab4d1bbacefce56057adad51d734f346702`; final closure is tracked in #1348. +- CL-03: **DRAFT PR OPEN** ([#1352](https://github.com/lidge-jun/opencodex/pull/1352)) on + `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`; + implementation head `003f7402f49bfe8dd710a7beba52f717051bfadf`. Not accepted. +- CL-04: **NOT STARTED** (blocked until CL-03 independent acceptance and review reconciliation). + +## CL-03 implementation log (2026-08-09) + +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting/base SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (#1348 merge on `dev`) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) - **Scope:** bounded live-route probes for `live_route_compatibility`; live manifest authority; `RouteSubjectV1` builder; `LabDestinationV1` / credential lease sandbox; inert tool/MCP stubs; live runner/executor; `observe/from-live` persistence; @@ -20,4 +189,4 @@ - Independent acceptance review not performed - Draft PR review findings not yet reconciled -- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) \ No newline at end of file +- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) From e0a07e29769091f3741835293e031849936aeac0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:44:45 +0200 Subject: [PATCH 38/47] fix(lab): harden salt publication durability --- src/lab/subject/installation-salt.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts index dfc804537..da2b42efc 100644 --- a/src/lab/subject/installation-salt.ts +++ b/src/lab/subject/installation-salt.ts @@ -20,6 +20,16 @@ function removeStagingFile(path: string): void { } } +function fsyncDirectory(path: string): void { + // Node does not provide a portable directory-fsync primitive on Windows. + // POSIX publication is crash-durable; Windows still uses a fsync'd staging inode + // plus atomic hard-link publication so partially written salts are never visible. + if (process.platform === "win32") return; + const dirFd = openSync(path, "r"); + try { fsyncSync(dirFd); } + finally { closeSync(dirFd); } +} + /** Read or atomically publish the per-installation salt used for local fingerprinting. */ export function readInstallationSalt(configDir?: string): Uint8Array { const path = labInstallationSaltPath(configDir); @@ -46,6 +56,8 @@ export function readInstallationSalt(configDir?: string): Uint8Array { try { linkSync(stagingPath, path); + try { fsyncDirectory(dirname(path)); } + catch { throw new Error("harness_failure: installation salt directory fsync failed"); } return new Uint8Array(salt); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; @@ -57,4 +69,4 @@ export function readInstallationSalt(configDir?: string): Uint8Array { } removeStagingFile(stagingPath); } -} +} \ No newline at end of file From 47ad32fbb007757924085c6433648177a63ca8da Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:45:12 +0200 Subject: [PATCH 39/47] fix(lab): type pinned connect timeouts --- src/lib/pinned-http.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib/pinned-http.ts b/src/lib/pinned-http.ts index 03c9f8f69..247c94581 100644 --- a/src/lib/pinned-http.ts +++ b/src/lib/pinned-http.ts @@ -3,6 +3,13 @@ import https from "node:https"; export type PinnedAddress = { address: string; family: number }; +export type PinnedHttpErrorCode = "connect_timeout"; + +export class PinnedHttpError extends Error { + override readonly name = "PinnedHttpError"; + constructor(readonly code: PinnedHttpErrorCode, message: string) { super(message); } +} + export interface PinnedHttpRequestOptions { headers?: HeadersInit; maxBytes?: number; @@ -158,7 +165,7 @@ function pinnedHttpRequest( req.on("socket", (socket) => { if (!socket.connecting || connectTimeoutMs === undefined) return; const connectedEvent = parsed.protocol === "https:" ? "secureConnect" : "connect"; - connectTimer = setTimeout(() => fail(new Error(`${context} connect timed out`)), connectTimeoutMs); + connectTimer = setTimeout(() => fail(new PinnedHttpError("connect_timeout", `${context} connect timed out`)), connectTimeoutMs); socket.once(connectedEvent, clearConnectTimer); socket.once("error", clearConnectTimer); socket.once("close", clearConnectTimer); @@ -201,4 +208,4 @@ export function pinnedHttpPost( options?: PinnedHttpRequestOptions, ): Promise { return pinnedHttpRequest(url, pinned, "POST", body, signal, options); -} +} \ No newline at end of file From c4e3ea9438144e5f3de8bba0793ef59c19d9b7ac Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:45:28 +0200 Subject: [PATCH 40/47] fix(lab): preserve pinned connect timeout code --- src/lib/lab-live-pinned-sender.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/lab-live-pinned-sender.ts b/src/lib/lab-live-pinned-sender.ts index 980bb6352..d25966d3a 100644 --- a/src/lib/lab-live-pinned-sender.ts +++ b/src/lib/lab-live-pinned-sender.ts @@ -1,4 +1,5 @@ -import { pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; +import { PinnedHttpError, pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; +import { TransportError } from "../lab/live/transport"; import type { LabCredentialLeaseV1, LabPinnedSender } from "../lab/live/types"; /** Only response metadata required by current live assertions crosses into Lab. */ @@ -22,9 +23,17 @@ export function createLabAuthorizedPinnedSender( rejectUnauthorized: true, context: "Lab provider response", }; - const response = request.method === "POST" - ? await pinnedHttpPost(url, pinned, request.body ?? "", signal, options) - : await pinnedHttpGet(url, pinned, signal, options); + let response: Response; + try { + response = request.method === "POST" + ? await pinnedHttpPost(url, pinned, request.body ?? "", signal, options) + : await pinnedHttpGet(url, pinned, signal, options); + } catch (error) { + if (error instanceof PinnedHttpError && error.code === "connect_timeout") { + throw new TransportError("connect_timeout", "pinned provider connection timed out"); + } + throw error; + } const body = await response.text(); const responseHeaders: Record = {}; for (const headerName of LAB_RESPONSE_HEADER_ALLOWLIST) { @@ -33,4 +42,4 @@ export function createLabAuthorizedPinnedSender( } return { status: response.status, headers: responseHeaders, body }; }; -} +} \ No newline at end of file From d53fde7ba8bd80443d1eaa6a2735bcb953d503a2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:45:54 +0200 Subject: [PATCH 41/47] fix(lab): persist live retry policy --- src/lab/observe/from-live.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index a34809024..f1a0baeb8 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -6,7 +6,7 @@ import type { FailureRecordV1, ObservationEvent } from "../events/types"; import { assignEventId } from "../events/validate"; import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; -import type { CaseAuthority, CaseRecord } from "../conformance/types"; +import type { CaseAuthority, CaseRecord, FailureRule } from "../conformance/types"; import { assertTrustedLiveResultReceipt } from "../live/executor"; import { expandLiveScenario } from "../live/manifest"; import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; @@ -36,17 +36,27 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom } } -function failureFromLiveResult(result: LiveScenarioRunResult): FailureRecordV1 | undefined { +function selectedFailureRule(result: LiveScenarioRunResult, authority: CaseAuthority): FailureRule | undefined { + const rules = authority.failureRuleSets[authority.manifestDefaults.failureRuleSet] ?? []; + return rules.find((rule) => + (result.transportError !== undefined && rule.match.includes(result.transportError)) + || (result.secondaryCode !== undefined && (rule.match.includes(result.secondaryCode) || rule.secondaryCode === result.secondaryCode)) + || rule.classification === result.classification, + ); +} + +function failureFromLiveResult(result: LiveScenarioRunResult, authority: CaseAuthority): FailureRecordV1 | undefined { if (result.passed || result.classification === "inconclusive") return undefined; const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" ? "harness" : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) ? "environment" : "route"; + const rule = selectedFailureRule(result, authority); return { class: result.classification, code: result.secondaryCode ?? result.classification, - retryable: false, + retryable: rule?.retry === "bounded", attribution, }; } @@ -78,7 +88,7 @@ export function observationFromLiveResult(result: LiveScenarioRunResult, caseRec assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, observedSummary: row.observedSummary, reason: row.reason })) } })); const subject = result.routeSubject; const subjectId = subjectIdForSubject(subject); const authorityLimits = authority.manifestDefaults.executionLimits; const limits: Record = {}; for (const key of OBSERVATION_LIMIT_NAMES) if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; - const failure = failureFromLiveResult(result); + const failure = failureFromLiveResult(result, authority); const eventWithoutId = { schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, evidenceLayer: "live_route_compatibility" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), scenarioManifestDigest: scenarioDigest, @@ -97,4 +107,4 @@ export function persistLiveResult(result: LiveScenarioRunResult, caseRecord: Cas const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { const { event } = observationFromLiveResult(result, caseRecord, authority, { ...opts, artifactStore: store }); appendLabEvent(paths.ledgerPath, event); return { event, ledgerPath: paths.ledgerPath }; } finally { if (ownsStore) store.close(); } -} +} \ No newline at end of file From 081c372c6bcd42b792a8d06d5b40be9c0b966413 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:47:55 +0200 Subject: [PATCH 42/47] fix(lab): bind trusted receipt to result payload --- src/lab/live/executor.ts | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index 5d1ae89dd..a8bbc9319 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -30,10 +30,37 @@ interface TrustedLiveResultReceipt { scenarioManifestDigest: string; suiteManifestDigest: string; routeSubjectId: string; + resultDigest: string; } const TRUSTED_RESULT_RECEIPTS = new WeakMap(); const LIVE_AUTHORITY_DOMAIN = "ocx-lab:live-authority:v1"; +const TRUSTED_LIVE_RESULT_DOMAIN = "ocx-lab:trusted-live-result:v1"; + +function trustedResultDigest(result: LiveScenarioRunResult): string { + const payload = { + scenarioId: result.scenarioId, + suite: result.suite, + startedAt: result.startedAt, + completedAt: result.completedAt, + passed: result.passed, + classification: result.classification, + secondaryCode: result.secondaryCode ?? null, + assertionResults: result.assertionResults.map((row) => ({ + id: row.id, + operator: row.operator, + required: row.required, + passed: row.passed, + observedSummary: row.observedSummary, + reason: row.reason ?? null, + })), + diagnostics: [...result.diagnostics], + routeSubject: result.routeSubject ?? null, + transportError: result.transportError ?? null, + executionAuthority: result.executionAuthority, + }; + return domainHash(TRUSTED_LIVE_RESULT_DOMAIN, jcsStringify(payload)); +} function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): TrustedLiveResultReceipt { if (!result.routeSubject) throw new Error("trusted live result has no route subject"); @@ -44,6 +71,7 @@ function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, autho scenarioManifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), suiteManifestDigest: suiteManifestDigest(liveSuiteManifestObjectForCase(caseRecord, authority)), routeSubjectId: subjectIdForSubject(result.routeSubject), + resultDigest: trustedResultDigest(result), }; } @@ -265,4 +293,4 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR if (error instanceof LabDestinationError) classified = { classification: error.code === "network_blocked" ? "network_failure" : "harness_failure", secondaryCode: error.code }; return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }); } -} +} \ No newline at end of file From 5d2b494f7e5817a1e5c5fdc4caa026cd2aa6fcc9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:48:55 +0200 Subject: [PATCH 43/47] test(lab): reject mutated trusted live results --- tests/lab-live-receipt-integrity.test.ts | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/lab-live-receipt-integrity.test.ts diff --git a/tests/lab-live-receipt-integrity.test.ts b/tests/lab-live-receipt-integrity.test.ts new file mode 100644 index 000000000..5d2883e70 --- /dev/null +++ b/tests/lab-live-receipt-integrity.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadLiveCaseAuthority, observationFromLiveResult, runLiveScenario } from "../src/lab"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; +import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; + +function behavior(): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: "openai-responses" }, + "wire.upstreamProtocol": { source: "lab_forced", value: "openai-responses" }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; +} + +function route(): LabRouteContext { + return { + providerId: "fixture-provider", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: [], + availableHarnessFeatures: ["live_transport"], + behaviorValues: behavior(), + }; +} + +function passingObservation(): NormalizedObservation { + return { + client: { + request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, + response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "OK" }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +test("trusted live receipt rejects post-seal outcome mutation", async () => { + const home = join(tmpdir(), `ocx-lab-receipt-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(home, { recursive: true, mode: 0o700 }); + try { + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((row) => row.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + routeExecutor: createHostIssuedLabRouteExecutor(async () => passingObservation()), + }); + expect(result.passed).toBe(true); + result.passed = false; + expect(() => observationFromLiveResult(result, scenario, authority, { configDir: home })) + .toThrow("trusted execution receipt mismatch"); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}); \ No newline at end of file From 4ee6cdab5453a4f71e0f3f627c1c02a0fc8ff8b2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:49:34 +0200 Subject: [PATCH 44/47] test(lab): use public IPv6 sandbox fixture --- tests/lab-live-sandbox.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts index 63831fc5b..26c189a54 100644 --- a/tests/lab-live-sandbox.test.ts +++ b/tests/lab-live-sandbox.test.ts @@ -55,8 +55,8 @@ describe("CL-03 live sandbox security seams", () => { }); test("address-set equality uses the same canonical representation on both sides", async () => { - const home = tempHome(); const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "2001:db8::1", family: 6 }, { address: "93.184.216.34", family: 4 }, { address: "93.184.216.34", family: 4 }], configDir: home }); - expect(() => assertDestinationAddressSet(destination, [{ address: "93.184.216.34", family: 4 }, { address: "2001:db8::1", family: 6 }])).not.toThrow(); + const home = tempHome(); const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 }, { address: "93.184.216.34", family: 4 }, { address: "93.184.216.34", family: 4 }], configDir: home }); + expect(() => assertDestinationAddressSet(destination, [{ address: "93.184.216.34", family: 4 }, { address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 }])).not.toThrow(); }); test("destination resolution obeys the connect timeout", async () => { @@ -71,7 +71,7 @@ describe("CL-03 live sandbox security seams", () => { test("redirects fail closed in mock transport", async () => { const transport = createMockTransport({ entries: [{ status: 302, body: "", headers: { location: "https://evil.example.com" } }] }); await expect(transport.request({ method: "GET", path: "/" })).rejects.toThrow(TransportError); }); test("credential lease is destination-bound, bounded, and non-serializable", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const lease = createCredentialLease({ destination, budget: 1 }); assertLeaseScope(lease, destination); expect(() => JSON.stringify(lease)).toThrow(LabCredentialError); lease.consume(); expect(() => lease.consume()).toThrow(LabCredentialError); }); - test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); + test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("connect_timeout", "connect")).classification).toBe("timeout"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); test("observable resource counters enforce child-process limit", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); }); test("untrusted runtime code cannot self-attest a trusted executor", async () => { @@ -99,4 +99,4 @@ describe("CL-03 live sandbox security seams", () => { test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); test("route subject stability across credential rotation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); createCredentialLease({ destination, budget: 1 }).consume(); const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); expect(subjectA).toEqual(subjectB); }); -}); +}); \ No newline at end of file From 44466771f5f8a3a5ee198ec2b628ee7e507e50d4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:52:33 +0200 Subject: [PATCH 45/47] fix(lab): tolerate unsupported directory fsync --- src/lab/subject/installation-salt.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts index da2b42efc..4f286ce67 100644 --- a/src/lab/subject/installation-salt.ts +++ b/src/lab/subject/installation-salt.ts @@ -4,6 +4,7 @@ import { dirname } from "node:path"; import { labInstallationSaltPath, labRoot } from "../paths"; const SALT_BYTES = 32; +const UNSUPPORTED_DIRECTORY_FSYNC_CODES = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]); function readSaltFile(path: string): Uint8Array { const bytes = readFileSync(path); @@ -21,13 +22,21 @@ function removeStagingFile(path: string): void { } function fsyncDirectory(path: string): void { - // Node does not provide a portable directory-fsync primitive on Windows. - // POSIX publication is crash-durable; Windows still uses a fsync'd staging inode - // plus atomic hard-link publication so partially written salts are never visible. + // Windows has no portable directory-fsync primitive. Some POSIX-backed + // filesystems (notably virtual/shared filesystems) report directory fsync as + // unsupported. In those cases the fsync'd staging inode plus atomic same-dir + // hard-link publication is the supported durability fallback. Unexpected I/O + // errors remain fatal. if (process.platform === "win32") return; const dirFd = openSync(path, "r"); - try { fsyncSync(dirFd); } - finally { closeSync(dirFd); } + try { + try { fsyncSync(dirFd); } + catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code && UNSUPPORTED_DIRECTORY_FSYNC_CODES.has(code)) return; + throw error; + } + } finally { closeSync(dirFd); } } /** Read or atomically publish the per-installation salt used for local fingerprinting. */ From 998de8ae3dea587b25491752e637fd9d1207ca2e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:53:28 +0200 Subject: [PATCH 46/47] fix(lab): seal exact failure retry policy --- src/lab/live/executor.ts | 47 ++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index a8bbc9319..fe71947a9 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -31,6 +31,7 @@ interface TrustedLiveResultReceipt { suiteManifestDigest: string; routeSubjectId: string; resultDigest: string; + retryPolicy: FailureRule["retry"] | null; } const TRUSTED_RESULT_RECEIPTS = new WeakMap(); @@ -62,7 +63,7 @@ function trustedResultDigest(result: LiveScenarioRunResult): string { return domainHash(TRUSTED_LIVE_RESULT_DOMAIN, jcsStringify(payload)); } -function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): TrustedLiveResultReceipt { +function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, retryPolicy: FailureRule["retry"] | null): TrustedLiveResultReceipt { if (!result.routeSubject) throw new Error("trusted live result has no route subject"); return { authorityDigest: domainHash(LIVE_AUTHORITY_DOMAIN, jcsStringify(authority)), @@ -72,11 +73,12 @@ function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, autho suiteManifestDigest: suiteManifestDigest(liveSuiteManifestObjectForCase(caseRecord, authority)), routeSubjectId: subjectIdForSubject(result.routeSubject), resultDigest: trustedResultDigest(result), + retryPolicy, }; } -function sealTrustedLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): void { - TRUSTED_RESULT_RECEIPTS.set(result, receiptFor(result, caseRecord, authority)); +function sealTrustedLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, retryPolicy: FailureRule["retry"] | null): void { + TRUSTED_RESULT_RECEIPTS.set(result, receiptFor(result, caseRecord, authority, retryPolicy)); } /** Verify the module-private execution receipt before any live result enters persistence. */ @@ -85,7 +87,7 @@ export function assertTrustedLiveResultReceipt(result: LiveScenarioRunResult, ca if (!actual || result.executionAuthority !== "trusted_route") { throw new Error("live result lacks trusted execution receipt"); } - const expected = receiptFor(result, caseRecord, authority); + const expected = receiptFor(result, caseRecord, authority, actual.retryPolicy); for (const key of Object.keys(expected) as Array) { if (actual[key] !== expected[key]) throw new Error("live result trusted execution receipt mismatch"); } @@ -94,6 +96,12 @@ export function assertTrustedLiveResultReceipt(result: LiveScenarioRunResult, ca } } +/** Return retryability sealed by the executor after validating the trusted receipt. */ +export function trustedLiveResultRetryable(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): boolean { + assertTrustedLiveResultReceipt(result, caseRecord, authority); + return TRUSTED_RESULT_RECEIPTS.get(result)?.retryPolicy === "bounded"; +} + function liveLimitsFromAuthority(authorityLimits: Record): LiveRunConfig { return { totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, @@ -123,10 +131,10 @@ function routePreconditionFailure(route: LabRouteContext, caseRecord: CaseRecord return null; } -function classifyWithFailureRules(rules: FailureRule[], signal: string): { classification: FailureClassification; secondaryCode: string } { +function classifyWithFailureRules(rules: FailureRule[], signal: string): { classification: FailureClassification; secondaryCode: string; retryPolicy: FailureRule["retry"] | null } { const rule = rules.find((row) => row.match.includes(signal)); - return rule ? { classification: rule.classification, secondaryCode: rule.secondaryCode ?? signal } - : { classification: "inconclusive", secondaryCode: "unclassified" }; + return rule ? { classification: rule.classification, secondaryCode: rule.secondaryCode ?? signal, retryPolicy: rule.retry } + : { classification: "inconclusive", secondaryCode: "unclassified", retryPolicy: null }; } function pathForProtocol(protocol: string): string { @@ -226,9 +234,10 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR let authority: CaseAuthority | undefined; let activeCase = caseRecord; let trustedExecutionStarted = false; - const complete = (partial: Omit): LiveScenarioRunResult => { + let failureRules: FailureRule[] = []; + const complete = (partial: Omit, retryPolicy: FailureRule["retry"] | null = null): LiveScenarioRunResult => { const result: LiveScenarioRunResult = { ...partial, executionAuthority, startedAt, completedAt: Math.max(startedAt, Date.now()) }; - if (trustedExecutionStarted && authority && result.routeSubject) sealTrustedLiveResult(result, activeCase, authority); + if (trustedExecutionStarted && authority && result.routeSubject) sealTrustedLiveResult(result, activeCase, authority, retryPolicy); return result; }; try { @@ -246,10 +255,11 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR if (preconditionFailure) { return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: "inconclusive", secondaryCode: preconditionFailure, assertionResults: [], diagnostics: [preconditionFailure], routeSubject }); } - const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); - routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); const expanded = expandLiveScenario(activeCase, authority); + failureRules = expanded.failureRules as FailureRule[]; const limits = liveLimitsFromAuthority(expanded.executionLimits as Record); + const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); + routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); const state = createSandboxResourceState(); let observation: NormalizedObservation; if (opts.routeExecutor) { @@ -275,14 +285,17 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR } const assertionResults = evaluateAssertions(activeCase.assertions, observation); const requiredFailures = assertionResults.filter((row) => row.required && !row.passed); - const failureRules = expanded.failureRules as FailureRule[]; if (activeCase.expectedFailure) { const matched = activeCase.expectedFailure.assertionIds.every((id) => assertionResults.find((row) => row.id === id)?.passed === true) && requiredFailures.length === 0; - return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: matched, classification: matched ? activeCase.expectedFailure.expectedClass : "protocol_failure", secondaryCode: matched ? activeCase.expectedFailure.expectedCode : "deterministic_assertion", assertionResults, diagnostics, routeSubject }); + if (matched) { + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: true, classification: activeCase.expectedFailure.expectedClass, secondaryCode: activeCase.expectedFailure.expectedCode, assertionResults, diagnostics, routeSubject }); + } + const mismatch = classifyWithFailureRules(failureRules, "required_assertion_failed"); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: mismatch.classification, secondaryCode: mismatch.secondaryCode, assertionResults, diagnostics, routeSubject }, mismatch.retryPolicy); } const passed = requiredFailures.length === 0; - const classified = passed ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass" } : classifyWithFailureRules(failureRules, "required_assertion_failed"); - return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }); + const classified = passed ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass", retryPolicy: null } : classifyWithFailureRules(failureRules, "required_assertion_failed"); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }, passed ? null : classified.retryPolicy); } catch (error) { const diagnosticCode = error instanceof TransportError || error instanceof LabSandboxError || error instanceof LabDestinationError ? error.code @@ -291,6 +304,8 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR let classified = classifyTransportError(error); if (error instanceof LabSandboxError) classified = { classification: error.code === "harness_failure" ? "harness_failure" : "budget_exhausted", secondaryCode: error.code }; if (error instanceof LabDestinationError) classified = { classification: error.code === "network_blocked" ? "network_failure" : "harness_failure", secondaryCode: error.code }; - return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }); + const failureSignal = error instanceof TransportError || error instanceof LabSandboxError || error instanceof LabDestinationError ? error.code : "harness_failure"; + const retryPolicy = failureRules.find((rule) => rule.match.includes(failureSignal))?.retry ?? null; + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }, retryPolicy); } } \ No newline at end of file From 7db3e4b3ac4771f4f87cac4445bedb91d493175e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:53:58 +0200 Subject: [PATCH 47/47] fix(lab): persist sealed retry decision --- src/lab/observe/from-live.ts | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index f1a0baeb8..c56f55191 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -6,8 +6,8 @@ import type { FailureRecordV1, ObservationEvent } from "../events/types"; import { assignEventId } from "../events/validate"; import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; -import type { CaseAuthority, CaseRecord, FailureRule } from "../conformance/types"; -import { assertTrustedLiveResultReceipt } from "../live/executor"; +import type { CaseAuthority, CaseRecord } from "../conformance/types"; +import { trustedLiveResultRetryable } from "../live/executor"; import { expandLiveScenario } from "../live/manifest"; import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; import type { LiveScenarioRunResult } from "../live/types"; @@ -36,27 +36,17 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom } } -function selectedFailureRule(result: LiveScenarioRunResult, authority: CaseAuthority): FailureRule | undefined { - const rules = authority.failureRuleSets[authority.manifestDefaults.failureRuleSet] ?? []; - return rules.find((rule) => - (result.transportError !== undefined && rule.match.includes(result.transportError)) - || (result.secondaryCode !== undefined && (rule.match.includes(result.secondaryCode) || rule.secondaryCode === result.secondaryCode)) - || rule.classification === result.classification, - ); -} - -function failureFromLiveResult(result: LiveScenarioRunResult, authority: CaseAuthority): FailureRecordV1 | undefined { +function failureFromLiveResult(result: LiveScenarioRunResult, retryable: boolean): FailureRecordV1 | undefined { if (result.passed || result.classification === "inconclusive") return undefined; const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" ? "harness" : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) ? "environment" : "route"; - const rule = selectedFailureRule(result, authority); return { class: result.classification, code: result.secondaryCode ?? result.classification, - retryable: rule?.retry === "bounded", + retryable, attribution, }; } @@ -68,7 +58,7 @@ function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveO } export function observationFromLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): { event: ObservationEvent; artifacts: ReturnType[] } { - assertTrustedLiveResultReceipt(result, caseRecord, authority); + const retryable = trustedLiveResultRetryable(result, caseRecord, authority); if (!result.routeSubject) throw new Error("live evidence without an exact RouteSubjectV1 is not persistable"); const { startedAt, completedAt } = requireExecutionTimes(result, opts); const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); @@ -88,7 +78,7 @@ export function observationFromLiveResult(result: LiveScenarioRunResult, caseRec assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, observedSummary: row.observedSummary, reason: row.reason })) } })); const subject = result.routeSubject; const subjectId = subjectIdForSubject(subject); const authorityLimits = authority.manifestDefaults.executionLimits; const limits: Record = {}; for (const key of OBSERVATION_LIMIT_NAMES) if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; - const failure = failureFromLiveResult(result, authority); + const failure = failureFromLiveResult(result, retryable); const eventWithoutId = { schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, evidenceLayer: "live_route_compatibility" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), scenarioManifestDigest: scenarioDigest,